You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/11/25 07:09:45 UTC
[1/8] kylin git commit: minor, trim build log [Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/KYLIN-2217-2 84a0726a5 -> 99197dafc (forced update)
minor, trim build log
Revert "KYLIN-2210 call CubeStatsReader.print at SaveStatisticsStep"
This reverts commit 3ab966b650debb83eb219c9ed8d357d141466776.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8530ebd6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8530ebd6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8530ebd6
Branch: refs/heads/KYLIN-2217-2
Commit: 8530ebd692b3362f7fb9eb07160c8c069d467fc4
Parents: 6afcb26
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Nov 25 09:52:47 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Nov 25 09:58:10 2016 +0800
----------------------------------------------------------------------
.../kylin/engine/mr/common/CubeStatsReader.java | 2 +-
.../kylin/engine/mr/steps/SaveStatisticsStep.java | 17 +++--------------
2 files changed, 4 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/8530ebd6/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
index c6839d6..c917cfb 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
@@ -227,7 +227,7 @@ public class CubeStatsReader {
return ret;
}
- public void print(PrintWriter out) {
+ private void print(PrintWriter out) {
Map<Long, Long> cuboidRows = getCuboidRowEstimatesHLL();
Map<Long, Double> cuboidSizes = getCuboidSizeMap();
List<Long> cuboids = new ArrayList<Long>(cuboidRows.keySet());
http://git-wip-us.apache.org/repos/asf/kylin/blob/8530ebd6/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
index 79346a5..020c62c 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
@@ -19,8 +19,6 @@
package org.apache.kylin.engine.mr.steps;
import java.io.IOException;
-import java.io.PrintWriter;
-import java.io.StringWriter;
import java.util.Random;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -31,8 +29,8 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.engine.mr.CubingJob;
-import org.apache.kylin.engine.mr.CubingJob.AlgorithmEnum;
import org.apache.kylin.engine.mr.HadoopUtil;
+import org.apache.kylin.engine.mr.CubingJob.AlgorithmEnum;
import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.engine.mr.common.CubeStatsReader;
import org.apache.kylin.job.exception.ExecuteException;
@@ -86,16 +84,6 @@ public class SaveStatisticsStep extends AbstractExecutable {
private void decideCubingAlgorithm(CubeSegment seg, KylinConfig kylinConf) throws IOException {
String algPref = kylinConf.getCubeAlgorithm();
-
- CubeStatsReader cubeStats = new CubeStatsReader(seg, kylinConf);
- StringWriter sw = new StringWriter();
- PrintWriter pw = new PrintWriter(sw);
- cubeStats.print(pw);
- pw.flush();
- pw.close();
- logger.info("Cube Stats Estimation for segment {} :", seg.toString());
- logger.info(sw.toString());
-
AlgorithmEnum alg;
if (AlgorithmEnum.INMEM.name().equalsIgnoreCase(algPref)) {
alg = AlgorithmEnum.INMEM;
@@ -115,13 +103,14 @@ public class SaveStatisticsStep extends AbstractExecutable {
} else if ("random".equalsIgnoreCase(algPref)) { // for testing
alg = new Random().nextBoolean() ? AlgorithmEnum.INMEM : AlgorithmEnum.LAYER;
} else { // the default
+ CubeStatsReader cubeStats = new CubeStatsReader(seg, kylinConf);
int mapperNumber = cubeStats.getMapperNumberOfFirstBuild();
int mapperNumLimit = kylinConf.getCubeAlgorithmAutoMapperLimit();
double mapperOverlapRatio = cubeStats.getMapperOverlapRatioOfFirstBuild();
double overlapThreshold = kylinConf.getCubeAlgorithmAutoThreshold();
logger.info("mapperNumber for " + seg + " is " + mapperNumber + " and threshold is " + mapperNumLimit);
logger.info("mapperOverlapRatio for " + seg + " is " + mapperOverlapRatio + " and threshold is " + overlapThreshold);
-
+
// in-mem cubing is good when
// 1) the cluster has enough mapper slots to run in parallel
// 2) the mapper overlap ratio is small, meaning the shuffle of in-mem MR has advantage
[7/8] kylin git commit: KYLIN-2220 Enforce same name between Cube &
CubeDesc
Posted by li...@apache.org.
KYLIN-2220 Enforce same name between Cube & CubeDesc
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b078dd9f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b078dd9f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b078dd9f
Branch: refs/heads/KYLIN-2217-2
Commit: b078dd9f1b8ce9b7d0dad85cc6c9aa55dd5d9b15
Parents: d367bf5
Author: kangkaisen <ka...@live.com>
Authored: Tue Nov 22 21:26:32 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Nov 25 15:05:36 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/cube/CubeManager.java | 19 +------------------
.../kylin/rest/controller/CubeController.java | 2 +-
.../apache/kylin/rest/service/CubeService.java | 8 --------
3 files changed, 2 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b078dd9f/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 9893040..307bb46 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -617,24 +617,6 @@ public class CubeManager implements IRealizationProvider {
}
}
- private long calculateStartOffsetForAppendSegment(CubeInstance cube) {
- List<CubeSegment> existing = cube.getSegments();
- if (existing.isEmpty()) {
- return 0;
- } else {
- return existing.get(existing.size() - 1).getSourceOffsetEnd();
- }
- }
-
- private long calculateStartDateForAppendSegment(CubeInstance cube) {
- List<CubeSegment> existing = cube.getSegments();
- if (existing.isEmpty()) {
- return cube.getDescriptor().getPartitionDateStart();
- } else {
- return existing.get(existing.size() - 1).getDateRangeEnd();
- }
- }
-
private void checkBuildingSegment(CubeInstance cube) {
int maxBuldingSeg = cube.getConfig().getMaxBuildingSegments();
if (cube.getBuildingSegments().size() >= maxBuldingSeg) {
@@ -923,6 +905,7 @@ public class CubeManager implements IRealizationProvider {
CubeDesc cubeDesc = CubeDescManager.getInstance(config).getCubeDesc(cube.getDescName());
checkNotNull(cubeDesc, "cube descriptor '%s' (for cube '%s') not found", cube.getDescName(), cubeName);
+ checkState(cubeDesc.getName().equals(cubeName), "cube name '%s' must be same as descriptor name '%s', but it is not", cubeName, cubeDesc.getName());
if (!cubeDesc.getError().isEmpty()) {
cube.setStatus(RealizationStatusEnum.DESCBROKEN);
http://git-wip-us.apache.org/repos/asf/kylin/blob/b078dd9f/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index e1aa17a..3846d28 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -416,7 +416,7 @@ public class CubeController extends BasicController {
cubeRequest.setMessage("CubeDesc is null.");
return cubeRequest;
}
- String name = CubeService.getCubeNameFromDesc(desc.getName());
+ String name = desc.getName();
if (StringUtils.isEmpty(name)) {
logger.info("Cube name should not be empty.");
throw new BadRequestException("Cube name should not be empty.");
http://git-wip-us.apache.org/repos/asf/kylin/blob/b078dd9f/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 5c59e1a..85c9284 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -278,14 +278,6 @@ public class CubeService extends BasicService {
accessService.clean(cube, true);
}
- public static String getCubeNameFromDesc(String descName) {
- if (descName.toLowerCase().endsWith(DESC_SUFFIX)) {
- return descName.substring(0, descName.toLowerCase().indexOf(DESC_SUFFIX));
- } else {
- return descName;
- }
- }
-
/**
* Stop all jobs belonging to this cube and clean out all segments
*
[4/8] kylin git commit: KYLIN-2217 fix bug
Posted by li...@apache.org.
KYLIN-2217 fix bug
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d9c8d9ec
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d9c8d9ec
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d9c8d9ec
Branch: refs/heads/KYLIN-2217-2
Commit: d9c8d9ec26b910f2a27f7a98dd44ef49e61fe416
Parents: 7f7417b
Author: Yang Li <li...@apache.org>
Authored: Fri Nov 25 07:16:45 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Nov 25 15:05:36 2016 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/engine/mr/DFSFileTable.java | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d9c8d9ec/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java
index 8c1f6bd..6f337d0 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java
@@ -61,12 +61,13 @@ public class DFSFileTable implements ReadableTable {
@Override
public TableSignature getSignature() throws IOException {
+ Pair<Long, Long> sizeAndLastModified;
try {
- Pair<Long, Long> sizeAndLastModified = getSizeAndLastModified(path);
- return new TableSignature(path, sizeAndLastModified.getFirst(), sizeAndLastModified.getSecond());
+ sizeAndLastModified = getSizeAndLastModified(path);
} catch (FileNotFoundException ex) {
- return null;
+ sizeAndLastModified = Pair.newPair(-1L, 0L);
}
+ return new TableSignature(path, sizeAndLastModified.getFirst(), sizeAndLastModified.getSecond());
}
@Override
[6/8] kylin git commit: KYLIN-2217 add saveDictionary() on
CubeManager & DictionaryManager
Posted by li...@apache.org.
KYLIN-2217 add saveDictionary() on CubeManager & DictionaryManager
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7f7417bd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7f7417bd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7f7417bd
Branch: refs/heads/KYLIN-2217-2
Commit: 7f7417bd2ed0eaa1766597d9dd4131cbe1d29757
Parents: b078dd9
Author: Li Yang <li...@apache.org>
Authored: Thu Nov 24 17:24:57 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Nov 25 15:05:36 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/cube/CubeManager.java | 29 ++++++---
.../kylin/cube/cli/DictionaryGeneratorCLI.java | 34 +++++++++-
.../org/apache/kylin/cube/util/CubingUtils.java | 2 +-
.../org/apache/kylin/dict/DictionaryInfo.java | 5 ++
.../apache/kylin/dict/DictionaryManager.java | 65 +++++++++-----------
.../engine/mr/steps/MergeCuboidMapperTest.java | 2 +-
.../kylin/cube/ITDictionaryManagerTest.java | 6 +-
7 files changed, 93 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f7417bd/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 307bb46..ea8ff81 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -48,7 +48,6 @@ import org.apache.kylin.cube.model.DictionaryDesc;
import org.apache.kylin.cube.model.DimensionDesc;
import org.apache.kylin.dict.DictionaryInfo;
import org.apache.kylin.dict.DictionaryManager;
-import org.apache.kylin.dict.DistinctColumnValuesProvider;
import org.apache.kylin.dict.lookup.LookupStringTable;
import org.apache.kylin.dict.lookup.SnapshotManager;
import org.apache.kylin.dict.lookup.SnapshotTable;
@@ -214,25 +213,39 @@ public class CubeManager implements IRealizationProvider {
return result;
}
- public DictionaryInfo buildDictionary(CubeSegment cubeSeg, TblColRef col, DistinctColumnValuesProvider factTableValueProvider) throws IOException {
+ public DictionaryInfo buildDictionary(CubeSegment cubeSeg, TblColRef col, ReadableTable inpTable) throws IOException {
CubeDesc cubeDesc = cubeSeg.getCubeDesc();
if (!cubeDesc.getAllColumnsNeedDictionaryBuilt().contains(col))
return null;
- DictionaryManager dictMgr = getDictionaryManager();
String builderClass = cubeDesc.getDictionaryBuilderClass(col);
- DictionaryInfo dictInfo = dictMgr.buildDictionary(cubeDesc.getModel(), col, factTableValueProvider, builderClass);
+ DictionaryInfo dictInfo = getDictionaryManager().buildDictionary(cubeDesc.getModel(), col, inpTable, builderClass);
+ saveDictionaryInfo(cubeSeg, col, dictInfo);
+ return dictInfo;
+ }
+
+ public DictionaryInfo saveDictionary(CubeSegment cubeSeg, TblColRef col, ReadableTable inpTable, Dictionary<String> dict) throws IOException {
+ CubeDesc cubeDesc = cubeSeg.getCubeDesc();
+ if (!cubeDesc.getAllColumnsNeedDictionaryBuilt().contains(col))
+ return null;
+
+ DictionaryInfo dictInfo = getDictionaryManager().saveDictionary(cubeDesc.getModel(), col, inpTable, dict);
+
+ saveDictionaryInfo(cubeSeg, col, dictInfo);
+ return dictInfo;
+ }
+
+ private void saveDictionaryInfo(CubeSegment cubeSeg, TblColRef col, DictionaryInfo dictInfo) throws IOException {
if (dictInfo != null) {
Dictionary<?> dict = dictInfo.getDictionaryObject();
cubeSeg.putDictResPath(col, dictInfo.getResourcePath());
cubeSeg.getRowkeyStats().add(new Object[] { col.getName(), dict.getSize(), dict.getSizeOfId() });
- CubeUpdate cubeBuilder = new CubeUpdate(cubeSeg.getCubeInstance());
- cubeBuilder.setToUpdateSegs(cubeSeg);
- updateCube(cubeBuilder);
+ CubeUpdate update = new CubeUpdate(cubeSeg.getCubeInstance());
+ update.setToUpdateSegs(cubeSeg);
+ updateCube(update);
}
- return dictInfo;
}
/**
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f7417bd/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
index 89e2e9b..a6aeb96 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
@@ -26,9 +26,15 @@ import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.model.DimensionDesc;
+import org.apache.kylin.dict.DictionaryManager;
import org.apache.kylin.dict.DistinctColumnValuesProvider;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.source.ReadableTable;
+import org.apache.kylin.source.SourceFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -51,7 +57,8 @@ public class DictionaryGeneratorCLI {
// dictionary
for (TblColRef col : cubeSeg.getCubeDesc().getAllColumnsNeedDictionaryBuilt()) {
logger.info("Building dictionary for " + col);
- cubeMgr.buildDictionary(cubeSeg, col, factTableValueProvider);
+ ReadableTable inpTable = decideInputTable(cubeSeg.getModel(), col, factTableValueProvider);
+ cubeMgr.buildDictionary(cubeSeg, col, inpTable);
}
// snapshot
@@ -67,4 +74,29 @@ public class DictionaryGeneratorCLI {
cubeMgr.buildSnapshotTable(cubeSeg, tableIdentity);
}
}
+
+ private static ReadableTable decideInputTable(DataModelDesc model, TblColRef col, DistinctColumnValuesProvider factTableValueProvider) {
+ KylinConfig config = model.getConfig();
+ DictionaryManager dictMgr = DictionaryManager.getInstance(config);
+ TblColRef srcCol = dictMgr.decideSourceData(model, col);
+ String srcTable = srcCol.getTable();
+
+ ReadableTable inpTable;
+ if (model.isFactTable(srcTable)) {
+ inpTable = factTableValueProvider.getDistinctValuesFor(srcCol);
+ } else {
+ MetadataManager metadataManager = MetadataManager.getInstance(config);
+ TableDesc tableDesc = new TableDesc(metadataManager.getTableDesc(srcTable));
+ if (TableDesc.TABLE_TYPE_VIRTUAL_VIEW.equalsIgnoreCase(tableDesc.getTableType())) {
+ TableDesc materializedTbl = new TableDesc();
+ materializedTbl.setDatabase(config.getHiveDatabaseForIntermediateTable());
+ materializedTbl.setName(tableDesc.getMaterializedName());
+ inpTable = SourceFactory.createReadableTable(materializedTbl);
+ } else {
+ inpTable = SourceFactory.createReadableTable(tableDesc);
+ }
+ }
+
+ return inpTable;
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f7417bd/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
index 10dad2c..413b907 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
@@ -170,7 +170,7 @@ public class CubingUtils {
signature.setLastModifiedTime(System.currentTimeMillis());
signature.setPath(String.format("streaming_%s_%s", startOffset, endOffset));
signature.setSize(endOffset - startOffset);
- DictionaryInfo dictInfo = new DictionaryInfo(tblColRef.getTable(), tblColRef.getName(), tblColRef.getColumnDesc().getZeroBasedIndex(), tblColRef.getDatatype(), signature);
+ DictionaryInfo dictInfo = new DictionaryInfo(tblColRef.getColumnDesc(), tblColRef.getDatatype(), signature);
logger.info("writing dictionary for TblColRef:" + tblColRef.toString());
DictionaryManager dictionaryManager = DictionaryManager.getInstance(cubeSegment.getCubeDesc().getConfig());
try {
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f7417bd/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
index 4fba59a..8526467 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
@@ -21,6 +21,7 @@ package org.apache.kylin.dict;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.source.ReadableTable.TableSignature;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
@@ -51,6 +52,10 @@ public class DictionaryInfo extends RootPersistentEntity {
public DictionaryInfo() {
}
+ public DictionaryInfo(ColumnDesc col, String dataType, TableSignature input) {
+ this(col.getTable().getIdentity(), col.getName(), col.getZeroBasedIndex(), dataType, input);
+ }
+
public DictionaryInfo(String sourceTable, String sourceColumn, int sourceColumnIndex, String dataType, TableSignature input) {
this.updateRandomUuid();
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f7417bd/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index c33cd28..37e4757 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -38,12 +38,10 @@ import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.JoinDesc;
-import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.source.ReadableTable;
import org.apache.kylin.source.ReadableTable.TableSignature;
-import org.apache.kylin.source.SourceFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -273,45 +271,19 @@ public class DictionaryManager {
}
}
- public DictionaryInfo buildDictionary(DataModelDesc model, TblColRef col, DistinctColumnValuesProvider factTableValueProvider) throws IOException {
- return buildDictionary(model, col, factTableValueProvider, null);
+ public DictionaryInfo buildDictionary(DataModelDesc model, TblColRef col, ReadableTable inpTable) throws IOException {
+ return buildDictionary(model, col, inpTable, null);
}
- public DictionaryInfo buildDictionary(DataModelDesc model, TblColRef col, DistinctColumnValuesProvider factTableValueProvider, String builderClass) throws IOException {
+ public DictionaryInfo buildDictionary(DataModelDesc model, TblColRef col, ReadableTable inpTable, String builderClass) throws IOException {
logger.info("building dictionary for " + col);
- TblColRef srcCol = decideSourceData(model, col);
- String srcTable = srcCol.getTable();
- String srcColName = srcCol.getName();
- int srcColIdx = srcCol.getColumnDesc().getZeroBasedIndex();
-
- ReadableTable inpTable;
- if (model.isFactTable(srcTable)) {
- inpTable = factTableValueProvider.getDistinctValuesFor(srcCol);
- } else {
- MetadataManager metadataManager = MetadataManager.getInstance(config);
- TableDesc tableDesc = new TableDesc(metadataManager.getTableDesc(srcTable));
- if (TableDesc.TABLE_TYPE_VIRTUAL_VIEW.equalsIgnoreCase(tableDesc.getTableType())) {
- TableDesc materializedTbl = new TableDesc();
- materializedTbl.setDatabase(config.getHiveDatabaseForIntermediateTable());
- materializedTbl.setName(tableDesc.getMaterializedName());
- inpTable = SourceFactory.createReadableTable(materializedTbl);
- } else {
- inpTable = SourceFactory.createReadableTable(tableDesc);
- }
- }
-
- TableSignature inputSig = inpTable.getSignature();
- if (inputSig == null) // table does not exists
- return null;
-
- DictionaryInfo dictInfo = new DictionaryInfo(srcTable, srcColName, srcColIdx, col.getDatatype(), inputSig);
-
- String dupDict = checkDupByInfo(dictInfo);
- if (dupDict != null) {
- logger.info("Identical dictionary input " + dictInfo.getInput() + ", reuse existing dictionary at " + dupDict);
- return getDictionaryInfo(dupDict);
+ DictionaryInfo dictInfo = createDictionaryInfo(model, col, inpTable);
+ String dupInfo = checkDupByInfo(dictInfo);
+ if (dupInfo != null) {
+ logger.info("Identical dictionary input " + dictInfo.getInput() + ", reuse existing dictionary at " + dupInfo);
+ return getDictionaryInfo(dupInfo);
}
logger.info("Building dictionary object " + JsonUtil.writeValueAsString(dictInfo));
@@ -333,6 +305,27 @@ public class DictionaryManager {
return trySaveNewDict(dictionary, dictInfo);
}
+ public DictionaryInfo saveDictionary(DataModelDesc model, TblColRef col, ReadableTable inpTable, Dictionary<String> dictionary) throws IOException {
+ DictionaryInfo dictInfo = createDictionaryInfo(model, col, inpTable);
+ String dupInfo = checkDupByInfo(dictInfo);
+ if (dupInfo != null) {
+ logger.info("Identical dictionary input " + dictInfo.getInput() + ", reuse existing dictionary at " + dupInfo);
+ return getDictionaryInfo(dupInfo);
+ }
+
+ return trySaveNewDict(dictionary, dictInfo);
+ }
+
+ private DictionaryInfo createDictionaryInfo(DataModelDesc model, TblColRef col, ReadableTable inpTable) throws IOException {
+ TblColRef srcCol = decideSourceData(model, col);
+ TableSignature inputSig = inpTable.getSignature();
+ if (inputSig == null) // table does not exists
+ throw new IllegalStateException("Input table does not exist: " + inpTable);
+
+ DictionaryInfo dictInfo = new DictionaryInfo(srcCol.getColumnDesc(), col.getDatatype(), inputSig);
+ return dictInfo;
+ }
+
/**
* Decide a dictionary's source data, leverage PK-FK relationship.
*/
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f7417bd/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java
index 6f3b7c9..075ec80 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java
@@ -119,7 +119,7 @@ public class MergeCuboidMapperTest extends LocalFileMetadataTestCase {
signature.setLastModifiedTime(System.currentTimeMillis());
signature.setPath("fake_dict_for" + lfn.getName() + segment.getName());
- DictionaryInfo newDictInfo = new DictionaryInfo(lfn.getTable(), lfn.getColumnDesc().getName(), lfn.getColumnDesc().getZeroBasedIndex(), "string", signature);
+ DictionaryInfo newDictInfo = new DictionaryInfo(lfn.getColumnDesc(), "string", signature);
List<String> values = new ArrayList<>();
values.add("aaa");
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f7417bd/kylin-it/src/test/java/org/apache/kylin/cube/ITDictionaryManagerTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/cube/ITDictionaryManagerTest.java b/kylin-it/src/test/java/org/apache/kylin/cube/ITDictionaryManagerTest.java
index 22ffd26..188a97a 100644
--- a/kylin-it/src/test/java/org/apache/kylin/cube/ITDictionaryManagerTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/cube/ITDictionaryManagerTest.java
@@ -65,10 +65,10 @@ public class ITDictionaryManagerTest extends LocalFileMetadataTestCase {
MockDistinctColumnValuesProvider mockupData = new MockDistinctColumnValuesProvider("A", "B", "C");
- DictionaryInfo info1 = dictMgr.buildDictionary(cubeDesc.getModel(), col, mockupData);
+ DictionaryInfo info1 = dictMgr.buildDictionary(cubeDesc.getModel(), col, mockupData.getDistinctValuesFor(col));
System.out.println(JsonUtil.writeValueAsIndentString(info1));
- DictionaryInfo info2 = dictMgr.buildDictionary(cubeDesc.getModel(), col, mockupData);
+ DictionaryInfo info2 = dictMgr.buildDictionary(cubeDesc.getModel(), col, mockupData.getDistinctValuesFor(col));
System.out.println(JsonUtil.writeValueAsIndentString(info2));
// test check duplicate
@@ -89,7 +89,7 @@ public class ITDictionaryManagerTest extends LocalFileMetadataTestCase {
// test empty dictionary
MockDistinctColumnValuesProvider mockupEmpty = new MockDistinctColumnValuesProvider();
- DictionaryInfo info3 = dictMgr.buildDictionary(cubeDesc.getModel(), col, mockupEmpty);
+ DictionaryInfo info3 = dictMgr.buildDictionary(cubeDesc.getModel(), col, mockupEmpty.getDistinctValuesFor(col));
System.out.println(JsonUtil.writeValueAsIndentString(info3));
assertEquals(0, info3.getCardinality());
assertEquals(0, info3.getDictionaryObject().getSize());
[8/8] kylin git commit: fix bug
Posted by li...@apache.org.
fix bug
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f700a429
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f700a429
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f700a429
Branch: refs/heads/KYLIN-2217-2
Commit: f700a42976b774e27fb5e5416075bfa21cefa153
Parents: d9c8d9e
Author: Li Yang <li...@apache.org>
Authored: Fri Nov 25 11:22:11 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Nov 25 15:05:36 2016 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/cube/CubeManager.java | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f700a429/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index ea8ff81..6a024b4 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -918,7 +918,8 @@ public class CubeManager implements IRealizationProvider {
CubeDesc cubeDesc = CubeDescManager.getInstance(config).getCubeDesc(cube.getDescName());
checkNotNull(cubeDesc, "cube descriptor '%s' (for cube '%s') not found", cube.getDescName(), cubeName);
- checkState(cubeDesc.getName().equals(cubeName), "cube name '%s' must be same as descriptor name '%s', but it is not", cubeName, cubeDesc.getName());
+ if (!isITTestCube(cubeName))
+ checkState(cubeDesc.getName().equals(cubeName), "cube name '%s' must be same as descriptor name '%s', but it is not", cubeName, cubeDesc.getName());
if (!cubeDesc.getError().isEmpty()) {
cube.setStatus(RealizationStatusEnum.DESCBROKEN);
@@ -947,6 +948,10 @@ public class CubeManager implements IRealizationProvider {
}
}
+ private boolean isITTestCube(String cubeName) {
+ return cubeName.startsWith("test_kylin_cube") || cubeName.startsWith("test_streaming");
+ }
+
private MetadataManager getMetadataManager() {
return MetadataManager.getInstance(config);
}
[5/8] kylin git commit: bug fix
Posted by li...@apache.org.
bug fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/99197daf
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/99197daf
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/99197daf
Branch: refs/heads/KYLIN-2217-2
Commit: 99197dafc3e33bca922e0ce711f54956169c77d5
Parents: f700a42
Author: Li Yang <li...@apache.org>
Authored: Fri Nov 25 11:25:08 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Nov 25 15:05:36 2016 +0800
----------------------------------------------------------------------
core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/99197daf/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 6a024b4..b4422d2 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -949,7 +949,8 @@ public class CubeManager implements IRealizationProvider {
}
private boolean isITTestCube(String cubeName) {
- return cubeName.startsWith("test_kylin_cube") || cubeName.startsWith("test_streaming");
+ return config.isDevEnv() //
+ && (cubeName.startsWith("test_kylin_cube") || cubeName.startsWith("test_streaming"));
}
private MetadataManager getMetadataManager() {
[3/8] kylin git commit: minor,
keep same column in Rowkey and AggrGroup
Posted by li...@apache.org.
minor, keep same column in Rowkey and AggrGroup
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d367bf56
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d367bf56
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d367bf56
Branch: refs/heads/KYLIN-2217-2
Commit: d367bf56df8cf6581cc3dbdae57b5ed167be47f7
Parents: 89875fa
Author: Li Yang <li...@apache.org>
Authored: Fri Nov 25 15:05:19 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Nov 25 15:05:19 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/cube/model/AggregationGroup.java | 2 +-
.../main/java/org/apache/kylin/cube/model/RowKeyColDesc.java | 2 +-
.../main/java/org/apache/kylin/metadata/model/TblColRef.java | 7 +++++++
3 files changed, 9 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d367bf56/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
index 12c0574..9bd082f 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
@@ -107,7 +107,7 @@ public class AggregationGroup {
for (int i = 0; i < names.length; i++) {
TblColRef col = cubeDesc.getModel().findColumn(names[i]);
- names[i] = col.getTableAlias() + "." + col.getName();
+ names[i] = col.getIdentity();
}
// check no dup
http://git-wip-us.apache.org/repos/asf/kylin/blob/d367bf56/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
index 326052e..3b49323 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
@@ -62,9 +62,9 @@ public class RowKeyColDesc {
private TblColRef colRef;
public void init(int index, CubeDesc cubeDesc) {
- column = column.toUpperCase();
bitIndex = index;
colRef = cubeDesc.getModel().findColumn(column);
+ column = colRef.getIdentity();
Preconditions.checkArgument(colRef != null, "Cannot find rowkey column %s in cube %s", column, cubeDesc);
Preconditions.checkState(StringUtils.isNotEmpty(this.encoding));
http://git-wip-us.apache.org/repos/asf/kylin/blob/d367bf56/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
index 9b71524..2cfbafc 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
@@ -99,6 +99,7 @@ public class TblColRef implements Serializable {
private TableRef table;
private ColumnDesc column;
+ private String identity;
TblColRef(ColumnDesc column) {
this.column = column;
@@ -188,6 +189,12 @@ public class TblColRef implements Serializable {
return true;
}
+ public String getIdentity() {
+ if (identity == null)
+ identity = getTableAlias() + "." + getName();
+ return identity;
+ }
+
@Override
public String toString() {
String alias = table == null ? "UNKNOWN_MODEL" : table.getAlias();
[2/8] kylin git commit: KYLIN-2195 re-format KylinConfigBase
Posted by li...@apache.org.
KYLIN-2195 re-format KylinConfigBase
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/89875fae
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/89875fae
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/89875fae
Branch: refs/heads/KYLIN-2217-2
Commit: 89875fae78556d4d5f5099accb70d6572faeb3d4
Parents: 8530ebd
Author: Li Yang <li...@apache.org>
Authored: Fri Nov 25 14:49:58 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Nov 25 14:49:58 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 778 ++++++++++---------
1 file changed, 411 insertions(+), 367 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/89875fae/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 6131013..8ea03be 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -142,16 +142,47 @@ abstract public class KylinConfigBase implements Serializable {
this.properties = BCC.check(properties);
}
+ private Map<Integer, String> convertKeyToInteger(Map<String, String> map) {
+ Map<Integer, String> result = Maps.newLinkedHashMap();
+ for (Entry<String, String> entry : map.entrySet()) {
+ result.put(Integer.valueOf(entry.getKey()), entry.getValue());
+ }
+ return result;
+ }
+
+ public String toString() {
+ return getMetadataUrl();
+ }
+
+ // ============================================================================
+ // ENV
// ============================================================================
public boolean isDevEnv() {
return "DEV".equals(getOptional("kylin.env", "DEV"));
}
+
+ public String getDeployEnv() {
+ return getOptional("kylin.env", "DEV");
+ }
+
+ public String getHdfsWorkingDirectory() {
+ String root = getRequired("kylin.env.hdfs-working-dir");
+ if (!root.endsWith("/")) {
+ root += "/";
+ }
+ return new StringBuffer(root).append(StringUtils.replaceChars(getMetadataUrlPrefix(), ':', '-')).append("/").toString();
+ }
+
+ // ============================================================================
+ // METADATA
+ // ============================================================================
public String getMetadataUrl() {
return getOptional("kylin.metadata.url");
}
+ // for test only
public void setMetadataUrl(String metadataUrl) {
setProperty("kylin.metadata.url", metadataUrl);
}
@@ -169,178 +200,120 @@ abstract public class KylinConfigBase implements Serializable {
}
}
- public String getServerMode() {
- return this.getOptional("kylin.server.mode", "all");
- }
-
- public String getStorageUrl() {
- return getOptional("kylin.storage.url");
- }
-
- public void setStorageUrl(String storageUrl) {
- setProperty("kylin.storage.url", storageUrl);
+ public String[] getRealizationProviders() {
+ return getOptionalStringArray("kylin.metadata.realization-providers", //
+ new String[] { "org.apache.kylin.cube.CubeManager", "org.apache.kylin.storage.hybrid.HybridManager" });
}
- /**
- * was for route to hive, not used any more
- */
- @Deprecated
- public String getHiveUrl() {
- return getOptional("kylin.source.hive.connection-url", "");
+ public String[] getCubeDimensionCustomEncodingFactories() {
+ return getOptionalStringArray("kylin.metadata.custom-dimension-encodings", new String[0]);
}
- /**
- * was for route to hive, not used any more
- */
- @Deprecated
- public String getHiveUser() {
- return getOptional("kylin.source.hive.connection-user", "");
+ public Map<String, String> getCubeCustomMeasureTypes() {
+ return getPropertiesByPrefix("kylin.metadata.custom-measure-types.");
}
+
+ // ============================================================================
+ // DICTIONARY & SNAPSHOT
+ // ============================================================================
- /**
- * was for route to hive, not used any more
- */
- @Deprecated
- public String getHivePassword() {
- return getOptional("kylin.source.hive.connection-password", "");
+ public int getTrieDictionaryForestMaxTrieSizeMB() {
+ return Integer.parseInt(getOptional("kylin.dictionary.forest-trie-max-mb", "500"));
}
- public String getHdfsWorkingDirectory() {
- String root = getRequired("kylin.env.hdfs-working-dir");
- if (!root.endsWith("/")) {
- root += "/";
- }
- return new StringBuffer(root).append(StringUtils.replaceChars(getMetadataUrlPrefix(), ':', '-')).append("/").toString();
+ public int getCachedDictMaxEntrySize() {
+ return Integer.parseInt(getOptional("kylin.dictionary.max-cache-entry", "3000"));
}
- public String[] getRealizationProviders() {
- return getOptionalStringArray("kylin.metadata.realization-providers", //
- new String[] { "org.apache.kylin.cube.CubeManager", "org.apache.kylin.storage.hybrid.HybridManager" });
+ public boolean isGrowingDictEnabled() {
+ return Boolean.parseBoolean(this.getOptional("kylin.dictionary.growing-enabled", "false"));
}
- public CliCommandExecutor getCliCommandExecutor() throws IOException {
- CliCommandExecutor exec = new CliCommandExecutor();
- if (getRunAsRemoteCommand()) {
- exec.setRunAtRemote(getRemoteHadoopCliHostname(), getRemoteHadoopCliPort(), getRemoteHadoopCliUsername(), getRemoteHadoopCliPassword());
- }
- return exec;
+ public int getAppendDictEntrySize() {
+ return Integer.parseInt(getOptional("kylin.dictionary.append-entry-size", "10000000"));
}
- public String getHBaseClusterFs() {
- return getOptional("kylin.storage.hbase.cluster-fs", "");
+ // for test
+ public void setAppendDictEntrySize(int entrySize) {
+ setProperty("kylin.dictionary.append-entry-size", String.valueOf(entrySize));
}
- public String getHBaseClusterHDFSConfigFile() {
- return getOptional("kylin.storage.hbase.cluster-hdfs-config-file", "");
+ public int getAppendDictCacheSize() {
+ return Integer.parseInt(getOptional("kylin.dictionary.append-cache-size", "20"));
}
- public String getKylinJobLogDir() {
- return getOptional("kylin.job.log-dir", "/tmp/kylin/logs");
+ // for test
+ public void setAppendDictCacheSize(int cacheSize) {
+ setProperty("kylin.dictionary.append-cache-size", String.valueOf(cacheSize));
}
- public String getKylinJobJarPath() {
- final String jobJar = getOptional("kylin.engine.mr.job-jar");
- if (StringUtils.isNotEmpty(jobJar)) {
- return jobJar;
- }
- String kylinHome = getKylinHome();
- if (StringUtils.isEmpty(kylinHome)) {
- return "";
- }
- return getFileName(kylinHome + File.separator + "lib", JOB_JAR_NAME_PATTERN);
+ public int getCachedSnapshotMaxEntrySize() {
+ return Integer.parseInt(getOptional("kylin.snapshot.max-cache-entry", "500"));
}
- public void overrideMRJobJarPath(String path) {
- logger.info("override " + "kylin.engine.mr.job-jar" + " to " + path);
- System.setProperty("kylin.engine.mr.job-jar", path);
+ public int getTableSnapshotMaxMB() {
+ return Integer.parseInt(getOptional("kylin.snapshot.max-mb", "300"));
}
- public String getKylinJobMRLibDir() {
- return getOptional("kylin.engine.mr.lib-dir", "");
- }
+ // ============================================================================
+ // CUBE
+ // ============================================================================
- public Map<String, String> getMRConfigOverride() {
- return getPropertiesByPrefix("kylin.engine.mr.config-override.");
+ public double getJobCuboidSizeRatio() {
+ return Double.parseDouble(getOptional("kylin.cube.size-estimate-ratio", "0.25"));
}
- public Map<String, String> getHiveConfigOverride() {
- return getPropertiesByPrefix("kylin.source.hive.config-override.");
+ public double getJobCuboidSizeMemHungryRatio() {
+ return Double.parseDouble(getOptional("kylin.cube.size-estimate-memhungry-ratio", "0.05"));
}
- public String getKylinSparkJobJarPath() {
- final String jobJar = getOptional("kylin.engine.mr.job-jar.spark");
- if (StringUtils.isNotEmpty(jobJar)) {
- return jobJar;
- }
- String kylinHome = getKylinHome();
- if (StringUtils.isEmpty(kylinHome)) {
- return "";
- }
- return getFileName(kylinHome + File.separator + "lib", SPARK_JOB_JAR_NAME_PATTERN);
+ public String getCubeAlgorithm() {
+ return getOptional("kylin.cube.algorithm", "auto");
}
- public void overrideSparkJobJarPath(String path) {
- logger.info("override " + "kylin.engine.mr.job-jar.spark" + " to " + path);
- System.setProperty("kylin.engine.mr.job-jar.spark", path);
+ public double getCubeAlgorithmAutoThreshold() {
+ return Double.parseDouble(getOptional("kylin.cube.algorithm.layer-or-inmem-threshold", "7"));
}
- private static final Pattern COPROCESSOR_JAR_NAME_PATTERN = Pattern.compile("kylin-coprocessor-(.+)\\.jar");
- private static final Pattern JOB_JAR_NAME_PATTERN = Pattern.compile("kylin-job-(.+)\\.jar");
- private static final Pattern SPARK_JOB_JAR_NAME_PATTERN = Pattern.compile("kylin-engine-spark-(.+)\\.jar");
-
- public String getCoprocessorLocalJar() {
- final String coprocessorJar = getOptional("kylin.storage.hbase.coprocessor-local-jar");
- if (StringUtils.isNotEmpty(coprocessorJar)) {
- return coprocessorJar;
- }
- String kylinHome = getKylinHome();
- if (StringUtils.isEmpty(kylinHome)) {
- throw new RuntimeException("getCoprocessorLocalJar needs KYLIN_HOME");
- }
- return getFileName(kylinHome + File.separator + "lib", COPROCESSOR_JAR_NAME_PATTERN);
+ public int getCubeAlgorithmAutoMapperLimit() {
+ return Integer.parseInt(getOptional("kylin.cube.algorithm.inmem-split-limit", "500"));
}
- public void overrideCoprocessorLocalJar(String path) {
- logger.info("override " + "kylin.storage.hbase.coprocessor-local-jar" + " to " + path);
- System.setProperty("kylin.storage.hbase.coprocessor-local-jar", path);
+ @Deprecated
+ public int getCubeAggrGroupMaxSize() {
+ return Integer.parseInt(getOptional("kylin.cube.aggrgroup.max-size", "12"));
}
- private static String getFileName(String homePath, Pattern pattern) {
- File home = new File(homePath);
- SortedSet<String> files = Sets.newTreeSet();
- if (home.exists() && home.isDirectory()) {
- for (File file : home.listFiles()) {
- final Matcher matcher = pattern.matcher(file.getName());
- if (matcher.matches()) {
- files.add(file.getAbsolutePath());
- }
- }
- }
- if (files.isEmpty()) {
- throw new RuntimeException("cannot find " + pattern.toString() + " in " + homePath);
- } else {
- return files.last();
- }
+ public int getCubeAggrGroupMaxCombination() {
+ return Integer.parseInt(getOptional("kylin.cube.aggrgroup.max-combination", "4096"));
}
- public double getDefaultHadoopJobReducerInputMB() {
- return Double.parseDouble(getOptional("kylin.engine.mr.reduce-input-mb", "500"));
+ public boolean getCubeAggrGroupIsMandatoryOnlyValid() {
+ return Boolean.parseBoolean(getOptional("kylin.cube.aggrgroup.is-mandatory-only-valid", "false"));
}
- public double getDefaultHadoopJobReducerCountRatio() {
- return Double.parseDouble(getOptional("kylin.engine.mr.reduce-count-ratio", "1.0"));
+ public int getMaxBuildingSegments() {
+ return Integer.parseInt(getOptional("kylin.cube.max-building-segments", "10"));
}
- public int getHadoopJobMinReducerNumber() {
- return Integer.parseInt(getOptional("kylin.engine.mr.min-reducer-number", "1"));
+ public void setMaxBuildingSegments(int maxBuildingSegments) {
+ setProperty("kylin.cube.max-building-segments", String.valueOf(maxBuildingSegments));
}
+
+ // ============================================================================
+ // JOB
+ // ============================================================================
- public int getHadoopJobMaxReducerNumber() {
- return Integer.parseInt(getOptional("kylin.engine.mr.max-reducer-number", "500"));
+ public CliCommandExecutor getCliCommandExecutor() throws IOException {
+ CliCommandExecutor exec = new CliCommandExecutor();
+ if (getRunAsRemoteCommand()) {
+ exec.setRunAtRemote(getRemoteHadoopCliHostname(), getRemoteHadoopCliPort(), getRemoteHadoopCliUsername(), getRemoteHadoopCliPassword());
+ }
+ return exec;
}
- public int getHadoopJobMapperInputRows() {
- return Integer.parseInt(getOptional("kylin.engine.mr.mapper-input-rows", "1000000"));
+ public String getKylinJobLogDir() {
+ return getOptional("kylin.job.log-dir", "/tmp/kylin/logs");
}
public boolean getRunAsRemoteCommand() {
@@ -387,177 +360,233 @@ abstract public class KylinConfigBase implements Serializable {
return Boolean.parseBoolean(getOptional("kylin.job.allow-empty-segment", "true"));
}
- //UHC: ultra high cardinality columns, contain the ShardByColumns and the GlobalDictionaryColumns
- public int getUHCReducerCount() {
- return Integer.parseInt(getOptional("kylin.engine.mr.uhc-reducer-count", "3"));
+ public int getMaxConcurrentJobLimit() {
+ return Integer.parseInt(getOptional("kylin.job.max-concurrent-jobs", "10"));
}
- public String getOverrideHiveTableLocation(String table) {
- return getOptional("kylin.source.hive.table-location." + table.toUpperCase());
+ public String[] getAdminDls() {
+ return getOptionalStringArray("kylin.job.notification-admin-emails", null);
}
- public String getYarnStatusCheckUrl() {
- return getOptional("kylin.engine.mr.yarn-check-status-url", null);
+ public int getCubingInMemSamplingPercent() {
+ int percent = Integer.parseInt(this.getOptional("kylin.job.sampling-percentage", "100"));
+ percent = Math.max(percent, 1);
+ percent = Math.min(percent, 100);
+ return percent;
}
- public int getYarnStatusCheckIntervalSeconds() {
- return Integer.parseInt(getOptional("kylin.engine.mr.yarn-check-interval-seconds", "60"));
+ public String getHiveDependencyFilterList() {
+ return this.getOptional("kylin.job.dependency-filter-list", "[^,]*hive-exec[0-9.-]+[^,]*?\\.jar" + "|" + "[^,]*hive-metastore[0-9.-]+[^,]*?\\.jar" + "|" + "[^,]*hive-hcatalog-core[0-9.-]+[^,]*?\\.jar");
}
- public int getMaxConcurrentJobLimit() {
- return Integer.parseInt(getOptional("kylin.job.max-concurrent-jobs", "10"));
+ public boolean isMailEnabled() {
+ return Boolean.parseBoolean(getOptional("kylin.job.notification-enabled", "false"));
}
- public String getTimeZone() {
- return getOptional("kylin.web.timezone", "PST");
+ public void setMailEnabled(boolean enable) {
+ setProperty("kylin.job.notification-enabled", "" + enable);
}
- public String[] getRestServers() {
- return getOptionalStringArray("kylin.server.cluster-servers", new String[0]);
+ public String getMailHost() {
+ return getOptional("kylin.job.notification-mail-host", "");
}
- public String getClusterName() {
- return this.getOptional("kylin.server.cluster-name", getMetadataUrlPrefix());
+ public String getMailUsername() {
+ return getOptional("kylin.job.notification-mail-username", "");
}
- public int getWorkersPerServer() {
- //for sequence sql use
- return Integer.parseInt(getOptional("kylin.server.sequence-sql.workers-per-server", "1"));
+ public String getMailPassword() {
+ return getOptional("kylin.job.notification-mail-password", "");
}
- public String[] getAdminDls() {
- return getOptionalStringArray("kylin.job.notification-admin-emails", null);
+ public String getMailSender() {
+ return getOptional("kylin.job.notification-mail-sender", "");
}
- public double getJobCuboidSizeRatio() {
- return Double.parseDouble(getOptional("kylin.cube.size-estimate-ratio", "0.25"));
+ public int getJobRetry() {
+ return Integer.parseInt(this.getOptional("kylin.job.retry", "0"));
}
- public double getJobCuboidSizeMemHungryRatio() {
- return Double.parseDouble(getOptional("kylin.cube.size-estimate-memhungry-ratio", "0.05"));
+ public int getCubeStatsHLLPrecision() {
+ return Integer.parseInt(getOptional("kylin.job.sampling-hll-precision", "14"));
}
- public String getCubeAlgorithm() {
- return getOptional("kylin.cube.algorithm", "auto");
+ public String getJobControllerLock() {
+ return getOptional("kylin.job.lock", "org.apache.kylin.storage.hbase.util.ZookeeperJobLock");
}
- public double getCubeAlgorithmAutoThreshold() {
- return Double.parseDouble(getOptional("kylin.cube.algorithm.layer-or-inmem-threshold", "7"));
+ public Map<Integer, String> getSchedulers() {
+ Map<Integer, String> r = convertKeyToInteger(getPropertiesByPrefix("kylin.job.scheduler.provider."));
+ r.put(0, "org.apache.kylin.job.impl.threadpool.DefaultScheduler");
+ r.put(2, "org.apache.kylin.job.impl.threadpool.DistributedScheduler");
+ return r;
}
- public int getCubeAlgorithmAutoMapperLimit() {
- return Integer.parseInt(getOptional("kylin.cube.algorithm.inmem-split-limit", "500"));
+ public Integer getSchedulerType() {
+ return Integer.parseInt(getOptional("kylin.job.scheduler.default", "0"));
}
- @Deprecated
- public int getCubeAggrGroupMaxSize() {
- return Integer.parseInt(getOptional("kylin.cube.aggrgroup.max-size", "12"));
- }
+ // ============================================================================
+ // SOURCE.HIVE
+ // ============================================================================
- public int getCubeAggrGroupMaxCombination() {
- return Integer.parseInt(getOptional("kylin.cube.aggrgroup.max-combination", "4096"));
+ public Map<Integer, String> getSourceEngines() {
+ Map<Integer, String> r = convertKeyToInteger(getPropertiesByPrefix("kylin.source.provider."));
+ // ref constants in ISourceAware
+ r.put(0, "org.apache.kylin.source.hive.HiveSource");
+ r.put(1, "org.apache.kylin.source.kafka.KafkaSource");
+ return r;
}
- public boolean getCubeAggrGroupIsMandatoryOnlyValid() {
- return Boolean.parseBoolean(getOptional("kylin.cube.aggrgroup.is-mandatory-only-valid", "false"));
+ /**
+ * was for route to hive, not used any more
+ */
+ @Deprecated
+ public String getHiveUrl() {
+ return getOptional("kylin.source.hive.connection-url", "");
}
- public String[] getCubeDimensionCustomEncodingFactories() {
- return getOptionalStringArray("kylin.metadata.custom-dimension-encodings", new String[0]);
+ /**
+ * was for route to hive, not used any more
+ */
+ @Deprecated
+ public String getHiveUser() {
+ return getOptional("kylin.source.hive.connection-user", "");
}
- public Map<String, String> getCubeCustomMeasureTypes() {
- return getPropertiesByPrefix("kylin.metadata.custom-measure-types.");
+ /**
+ * was for route to hive, not used any more
+ */
+ @Deprecated
+ public String getHivePassword() {
+ return getOptional("kylin.source.hive.connection-password", "");
}
- public int getTableSnapshotMaxMB() {
- return Integer.parseInt(getOptional("kylin.snapshot.max-mb", "300"));
+ public Map<String, String> getHiveConfigOverride() {
+ return getPropertiesByPrefix("kylin.source.hive.config-override.");
}
- public int getTrieDictionaryForestMaxTrieSizeMB() {
- return Integer.parseInt(getOptional("kylin.dictionary.forest-trie-max-mb", "500"));
+ public String getOverrideHiveTableLocation(String table) {
+ return getOptional("kylin.source.hive.table-location." + table.toUpperCase());
}
- public int getHBaseRegionCountMin() {
- return Integer.parseInt(getOptional("kylin.storage.hbase.min-region-count", "1"));
+ public boolean isHiveKeepFlatTable() {
+ return Boolean.parseBoolean(this.getOptional("kylin.source.hive.keep-flat-table", "false"));
}
- public int getHBaseRegionCountMax() {
- return Integer.parseInt(getOptional("kylin.storage.hbase.max-region-count", "500"));
+ public String getHiveDatabaseForIntermediateTable() {
+ return this.getOptional("kylin.source.hive.database-for-flat-table", "default");
}
- // for test only
- public void setHBaseHFileSizeGB(float size) {
- setProperty("kylin.storage.hbase.hfile-size-gb", String.valueOf(size));
+ public boolean isHiveRedistributeEnabled() {
+ return Boolean.parseBoolean(this.getOptional("kylin.source.hive.redistribute-flat-table", "true"));
}
- public float getHBaseHFileSizeGB() {
- return Float.parseFloat(getOptional("kylin.storage.hbase.hfile-size-gb", "2.0"));
+ public String getHiveClientMode() {
+ return getOptional("kylin.source.hive.client", "cli");
}
- //check KYLIN-1684, in most cases keep the default value
- public boolean isSkippingEmptySegments() {
- return Boolean.valueOf(getOptional("kylin.query.skip-empty-segments", "true"));
+ public String getHiveBeelineParams() {
+ return getOptional("kylin.source.hive.beeline-params", "");
}
- public int getStoragePushDownLimitMax() {
- return Integer.parseInt(getOptional("kylin.query.max-limit-pushdown", "10000"));
+ @Deprecated
+ public String getCreateFlatHiveTableMethod() {
+ return getOptional("kylin.source.hive.create-flat-table-method", "1");
}
- public int getScanThreshold() {
- return Integer.parseInt(getOptional("kylin.query.scan-threshold", "10000000"));
+ // ============================================================================
+ // STORAGE.HBASE
+ // ============================================================================
+
+ public Map<Integer, String> getStorageEngines() {
+ Map<Integer, String> r = convertKeyToInteger(getPropertiesByPrefix("kylin.storage.provider."));
+ // ref constants in IStorageAware
+ r.put(0, "org.apache.kylin.storage.hbase.HBaseStorage");
+ r.put(1, "org.apache.kylin.storage.hybrid.HybridStorage");
+ r.put(2, "org.apache.kylin.storage.hbase.HBaseStorage");
+ return r;
}
- public int getDerivedInThreshold() {
- return Integer.parseInt(getOptional("kylin.query.derived-filter-translation-threshold", "20"));
+ public int getDefaultStorageEngine() {
+ return Integer.parseInt(getOptional("kylin.storage.default", "2"));
}
- public int getBadQueryStackTraceDepth() {
- return Integer.parseInt(getOptional("kylin.query.badquery-stacktrace-depth", "10"));
+ public String getStorageUrl() {
+ return getOptional("kylin.storage.url");
}
- public int getBadQueryHistoryNum() {
- return Integer.parseInt(getOptional("kylin.query.badquery-history-number", "10"));
+ // for test only
+ public void setStorageUrl(String storageUrl) {
+ setProperty("kylin.storage.url", storageUrl);
}
- public int getBadQueryDefaultAlertingSeconds() {
- return Integer.parseInt(getOptional("kylin.query.badquery-alerting-seconds", "90"));
+ public String getHBaseClusterFs() {
+ return getOptional("kylin.storage.hbase.cluster-fs", "");
}
- public int getBadQueryDefaultDetectIntervalSeconds() {
- return Integer.parseInt(getOptional("kylin.query.badquery-detect-interval", "60"));
+ public String getHBaseClusterHDFSConfigFile() {
+ return getOptional("kylin.storage.hbase.cluster-hdfs-config-file", "");
}
- public boolean getBadQueryPersistentEnabled() {
- return Boolean.parseBoolean(getOptional("kylin.query.badquery-persistent-enabled", "true"));
+ private static final Pattern COPROCESSOR_JAR_NAME_PATTERN = Pattern.compile("kylin-coprocessor-(.+)\\.jar");
+ private static final Pattern JOB_JAR_NAME_PATTERN = Pattern.compile("kylin-job-(.+)\\.jar");
+ private static final Pattern SPARK_JOB_JAR_NAME_PATTERN = Pattern.compile("kylin-engine-spark-(.+)\\.jar");
+
+ public String getCoprocessorLocalJar() {
+ final String coprocessorJar = getOptional("kylin.storage.hbase.coprocessor-local-jar");
+ if (StringUtils.isNotEmpty(coprocessorJar)) {
+ return coprocessorJar;
+ }
+ String kylinHome = getKylinHome();
+ if (StringUtils.isEmpty(kylinHome)) {
+ throw new RuntimeException("getCoprocessorLocalJar needs KYLIN_HOME");
+ }
+ return getFileName(kylinHome + File.separator + "lib", COPROCESSOR_JAR_NAME_PATTERN);
}
- public String[] getQueryTransformers() {
- return getOptionalStringArray("kylin.query.transformers", new String[0]);
+ public void overrideCoprocessorLocalJar(String path) {
+ logger.info("override " + "kylin.storage.hbase.coprocessor-local-jar" + " to " + path);
+ System.setProperty("kylin.storage.hbase.coprocessor-local-jar", path);
}
- public int getCachedDictMaxEntrySize() {
- return Integer.parseInt(getOptional("kylin.dictionary.max-cache-entry", "3000"));
+ private static String getFileName(String homePath, Pattern pattern) {
+ File home = new File(homePath);
+ SortedSet<String> files = Sets.newTreeSet();
+ if (home.exists() && home.isDirectory()) {
+ for (File file : home.listFiles()) {
+ final Matcher matcher = pattern.matcher(file.getName());
+ if (matcher.matches()) {
+ files.add(file.getAbsolutePath());
+ }
+ }
+ }
+ if (files.isEmpty()) {
+ throw new RuntimeException("cannot find " + pattern.toString() + " in " + homePath);
+ } else {
+ return files.last();
+ }
}
- public int getCachedSnapshotMaxEntrySize() {
- return Integer.parseInt(getOptional("kylin.snapshot.max-cache-entry", "500"));
+ public int getHBaseRegionCountMin() {
+ return Integer.parseInt(getOptional("kylin.storage.hbase.min-region-count", "1"));
}
- public boolean getQueryRunLocalCoprocessor() {
- return Boolean.parseBoolean(getOptional("kylin.storage.hbase.run-local-coprocessor", "false"));
+ public int getHBaseRegionCountMax() {
+ return Integer.parseInt(getOptional("kylin.storage.hbase.max-region-count", "500"));
}
- public long getQueryDurationCacheThreshold() {
- return Long.parseLong(this.getOptional("kylin.query.cache-threshold-duration", String.valueOf(2000)));
+ // for test only
+ public void setHBaseHFileSizeGB(float size) {
+ setProperty("kylin.storage.hbase.hfile-size-gb", String.valueOf(size));
}
- public long getQueryScanCountCacheThreshold() {
- return Long.parseLong(this.getOptional("kylin.query.cache-threshold-scan-count", String.valueOf(10 * 1024)));
+ public float getHBaseHFileSizeGB() {
+ return Float.parseFloat(getOptional("kylin.storage.hbase.hfile-size-gb", "2.0"));
}
- public long getQueryMemBudget() {
- return Long.parseLong(this.getOptional("kylin.query.memory-budget-bytes", String.valueOf(3L * 1024 * 1024 * 1024)));
+ public boolean getQueryRunLocalCoprocessor() {
+ return Boolean.parseBoolean(getOptional("kylin.storage.hbase.run-local-coprocessor", "false"));
}
public double getQueryCoprocessorMemGB() {
@@ -568,18 +597,6 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.parseInt(this.getOptional("kylin.storage.hbase.coprocessor-timeout-seconds", "0"));
}
- public boolean isQuerySecureEnabled() {
- return Boolean.parseBoolean(this.getOptional("kylin.query.security-enabled", "true"));
- }
-
- public boolean isQueryCacheEnabled() {
- return Boolean.parseBoolean(this.getOptional("kylin.query.cache-enabled", "true"));
- }
-
- public boolean isQueryIgnoreUnknownFunction() {
- return Boolean.parseBoolean(this.getOptional("kylin.query.ignore-unknown-function", "false"));
- }
-
public int getQueryScanFuzzyKeyMax() {
return Integer.parseInt(this.getOptional("kylin.storage.hbase.max-fuzzykey-scan", "200"));
}
@@ -588,23 +605,6 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.valueOf(this.getOptional("kylin.storage.hbase.max-visit-scanrange", "1000000"));
}
- public String getQueryAccessController() {
- return getOptional("kylin.query.access-controller", null);
- }
-
- public long getSequenceExpireTime() {
- return Long.valueOf(this.getOptional("kylin.server.sequence-sql.expire-time", "86400000"));//default a day
- }
-
- public boolean getQueryMetricsEnabled() {
- return Boolean.parseBoolean(getOptional("kylin.server.query-metrics-enabled", "false"));
- }
-
- public int[] getQueryMetricsPercentilesIntervals() {
- String[] dft = { "60", "300", "3600" };
- return getOptionalIntArray("kylin.server.query-metrics-percentiles-intervals", dft);
- }
-
public String getDefaultIGTStorage() {
return getOptional("kylin.storage.hbase.gtstorage", "org.apache.kylin.storage.hbase.cube.v2.CubeHBaseEndpointRPC");
}
@@ -613,15 +613,6 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.parseInt(this.getOptional("kylin.storage.hbase.scan-cache-rows", "1024"));
}
- public boolean isGrowingDictEnabled() {
- return Boolean.parseBoolean(this.getOptional("kylin.dictionary.growing-enabled", "false"));
- }
-
- /**
- * HBase region cut size, in GB
- *
- * @return
- */
public float getKylinHBaseRegionCut() {
return Float.valueOf(getOptional("kylin.storage.hbase.region-cut-gb", "5.0"));
}
@@ -630,13 +621,6 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.parseInt(this.getOptional("kylin.storage.hbase.max-scan-result-bytes", "" + (5 * 1024 * 1024))); // 5 MB
}
- public int getCubingInMemSamplingPercent() {
- int percent = Integer.parseInt(this.getOptional("kylin.job.sampling-percentage", "100"));
- percent = Math.max(percent, 1);
- percent = Math.min(percent, 100);
- return percent;
- }
-
public String getHbaseDefaultCompressionCodec() {
return getOptional("kylin.storage.hbase.compression-codec", "none");
}
@@ -653,152 +637,204 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.valueOf(getOptional("kylin.storage.hbase.small-family-block-size-bytes", "65536"));
}
- public boolean isHiveKeepFlatTable() {
- return Boolean.parseBoolean(this.getOptional("kylin.source.hive.keep-flat-table", "false"));
+ public String getKylinOwner() {
+ return this.getOptional("kylin.storage.hbase.owner-tag", "");
}
- public String getHiveDatabaseForIntermediateTable() {
- return this.getOptional("kylin.source.hive.database-for-flat-table", "default");
+ public boolean getCompressionResult() {
+ return Boolean.parseBoolean(getOptional("kylin.storage.hbase.endpoint-compress-result", "true"));
}
- public boolean isHiveRedistributeEnabled() {
- return Boolean.parseBoolean(this.getOptional("kylin.source.hive.redistribute-flat-table", "true"));
+ public int getHBaseMaxConnectionThreads() {
+ return Integer.parseInt(getOptional("kylin.storage.hbase.max-hconnection-threads", "2048"));
}
- public String getHiveDependencyFilterList() {
- return this.getOptional("kylin.job.dependency-filter-list", "[^,]*hive-exec[0-9.-]+[^,]*?\\.jar" + "|" + "[^,]*hive-metastore[0-9.-]+[^,]*?\\.jar" + "|" + "[^,]*hive-hcatalog-core[0-9.-]+[^,]*?\\.jar");
+ public int getHBaseCoreConnectionThreads() {
+ return Integer.parseInt(getOptional("kylin.storage.hbase.core-hconnection-threads", "2048"));
}
- public String getKylinOwner() {
- return this.getOptional("kylin.storage.hbase.owner-tag", "");
+ public long getHBaseConnectionThreadPoolAliveSeconds() {
+ return Long.parseLong(getOptional("kylin.storage.hbase.hconnection-threads-alive-seconds", "60"));
}
- public String getSparkHome() {
- return getRequired("kylin.engine.spark.spark-home");
+ // ============================================================================
+ // ENGINE.MR
+ // ============================================================================
+
+ public Map<Integer, String> getJobEngines() {
+ Map<Integer, String> r = convertKeyToInteger(getPropertiesByPrefix("kylin.engine.provider."));
+ // ref constants in IEngineAware
+ r.put(0, "org.apache.kylin.engine.mr.MRBatchCubingEngine");
+ r.put(2, "org.apache.kylin.engine.mr.MRBatchCubingEngine2");
+ return r;
}
- public String getSparkMaster() {
- return getRequired("kylin.engine.spark.spark-master");
+ public int getDefaultCubeEngine() {
+ return Integer.parseInt(getOptional("kylin.engine.default", "2"));
}
- public boolean isMailEnabled() {
- return Boolean.parseBoolean(getOptional("kylin.job.notification-enabled", "false"));
+ public String getKylinJobJarPath() {
+ final String jobJar = getOptional("kylin.engine.mr.job-jar");
+ if (StringUtils.isNotEmpty(jobJar)) {
+ return jobJar;
+ }
+ String kylinHome = getKylinHome();
+ if (StringUtils.isEmpty(kylinHome)) {
+ return "";
+ }
+ return getFileName(kylinHome + File.separator + "lib", JOB_JAR_NAME_PATTERN);
}
- public void setMailEnabled(boolean enable) {
- setProperty("kylin.job.notification-enabled", "" + enable);
+ public void overrideMRJobJarPath(String path) {
+ logger.info("override " + "kylin.engine.mr.job-jar" + " to " + path);
+ System.setProperty("kylin.engine.mr.job-jar", path);
}
- public String getMailHost() {
- return getOptional("kylin.job.notification-mail-host", "");
+ public String getKylinJobMRLibDir() {
+ return getOptional("kylin.engine.mr.lib-dir", "");
}
- public String getMailUsername() {
- return getOptional("kylin.job.notification-mail-username", "");
+ public Map<String, String> getMRConfigOverride() {
+ return getPropertiesByPrefix("kylin.engine.mr.config-override.");
}
- public String getMailPassword() {
- return getOptional("kylin.job.notification-mail-password", "");
+ public double getDefaultHadoopJobReducerInputMB() {
+ return Double.parseDouble(getOptional("kylin.engine.mr.reduce-input-mb", "500"));
}
- public String getMailSender() {
- return getOptional("kylin.job.notification-mail-sender", "");
+ public double getDefaultHadoopJobReducerCountRatio() {
+ return Double.parseDouble(getOptional("kylin.engine.mr.reduce-count-ratio", "1.0"));
}
- public boolean isWebCrossDomainEnabled() {
- return Boolean.parseBoolean(getOptional("kylin.web.cross-domain-enabled", "true"));
+ public int getHadoopJobMinReducerNumber() {
+ return Integer.parseInt(getOptional("kylin.engine.mr.min-reducer-number", "1"));
}
- public int getJobRetry() {
- return Integer.parseInt(this.getOptional("kylin.job.retry", "0"));
+ public int getHadoopJobMaxReducerNumber() {
+ return Integer.parseInt(getOptional("kylin.engine.mr.max-reducer-number", "500"));
}
- public String toString() {
- return getMetadataUrl();
+ public int getHadoopJobMapperInputRows() {
+ return Integer.parseInt(getOptional("kylin.engine.mr.mapper-input-rows", "1000000"));
}
- public String getHiveClientMode() {
- return getOptional("kylin.source.hive.client", "cli");
+ //UHC: ultra high cardinality columns, contain the ShardByColumns and the GlobalDictionaryColumns
+ public int getUHCReducerCount() {
+ return Integer.parseInt(getOptional("kylin.engine.mr.uhc-reducer-count", "3"));
}
- public String getHiveBeelineParams() {
- return getOptional("kylin.source.hive.beeline-params", "");
+ public String getYarnStatusCheckUrl() {
+ return getOptional("kylin.engine.mr.yarn-check-status-url", null);
}
- public String getDeployEnv() {
- return getOptional("kylin.env", "DEV");
+ public int getYarnStatusCheckIntervalSeconds() {
+ return Integer.parseInt(getOptional("kylin.engine.mr.yarn-check-interval-seconds", "60"));
}
- public String getInitTasks() {
- return getOptional("kylin.server.init-tasks");
+ // ============================================================================
+ // ENGINE.SPARK
+ // ============================================================================
+
+ public String getKylinSparkJobJarPath() {
+ final String jobJar = getOptional("kylin.engine.spark.job-jar");
+ if (StringUtils.isNotEmpty(jobJar)) {
+ return jobJar;
+ }
+ String kylinHome = getKylinHome();
+ if (StringUtils.isEmpty(kylinHome)) {
+ return "";
+ }
+ return getFileName(kylinHome + File.separator + "lib", SPARK_JOB_JAR_NAME_PATTERN);
}
- public int getDimCountDistinctMaxCardinality() {
- return Integer.parseInt(getOptional("kylin.query.max-dimension-count-distinct", "5000000"));
+ public void overrideSparkJobJarPath(String path) {
+ logger.info("override " + "kylin.engine.spark.job-jar" + " to " + path);
+ System.setProperty("kylin.engine.spark.job-jar", path);
}
- public int getCubeStatsHLLPrecision() {
- return Integer.parseInt(getOptional("kylin.job.sampling-hll-precision", "14"));
+ public String getSparkHome() {
+ return getRequired("kylin.engine.spark.spark-home");
}
- public String getJobControllerLock() {
- return getOptional("kylin.job.lock", "org.apache.kylin.storage.hbase.util.ZookeeperJobLock");
+ public String getSparkMaster() {
+ return getRequired("kylin.engine.spark.spark-master");
}
- public Map<Integer, String> getJobEngines() {
- Map<Integer, String> r = convertKeyToInteger(getPropertiesByPrefix("kylin.engine.provider."));
- // ref constants in IEngineAware
- r.put(0, "org.apache.kylin.engine.mr.MRBatchCubingEngine");
- r.put(2, "org.apache.kylin.engine.mr.MRBatchCubingEngine2");
- return r;
+ // ============================================================================
+ // QUERY
+ // ============================================================================
+
+ //check KYLIN-1684, in most cases keep the default value
+ public boolean isSkippingEmptySegments() {
+ return Boolean.valueOf(getOptional("kylin.query.skip-empty-segments", "true"));
}
- public Map<Integer, String> getSourceEngines() {
- Map<Integer, String> r = convertKeyToInteger(getPropertiesByPrefix("kylin.source.provider."));
- // ref constants in ISourceAware
- r.put(0, "org.apache.kylin.source.hive.HiveSource");
- r.put(1, "org.apache.kylin.source.kafka.KafkaSource");
- return r;
+ public int getStoragePushDownLimitMax() {
+ return Integer.parseInt(getOptional("kylin.query.max-limit-pushdown", "10000"));
}
- public Map<Integer, String> getStorageEngines() {
- Map<Integer, String> r = convertKeyToInteger(getPropertiesByPrefix("kylin.storage.provider."));
- // ref constants in IStorageAware
- r.put(0, "org.apache.kylin.storage.hbase.HBaseStorage");
- r.put(1, "org.apache.kylin.storage.hybrid.HybridStorage");
- r.put(2, "org.apache.kylin.storage.hbase.HBaseStorage");
- return r;
+ public int getScanThreshold() {
+ return Integer.parseInt(getOptional("kylin.query.scan-threshold", "10000000"));
}
- public int getDefaultStorageEngine() {
- return Integer.parseInt(getOptional("kylin.storage.default", "2"));
+ public int getDerivedInThreshold() {
+ return Integer.parseInt(getOptional("kylin.query.derived-filter-translation-threshold", "20"));
}
- public int getDefaultCubeEngine() {
- return Integer.parseInt(getOptional("kylin.engine.default", "2"));
+ public int getBadQueryStackTraceDepth() {
+ return Integer.parseInt(getOptional("kylin.query.badquery-stacktrace-depth", "10"));
}
- public Map<Integer, String> getSchedulers() {
- Map<Integer, String> r = convertKeyToInteger(getPropertiesByPrefix("kylin.job.scheduler.provider."));
- r.put(0, "org.apache.kylin.job.impl.threadpool.DefaultScheduler");
- r.put(2, "org.apache.kylin.job.impl.threadpool.DistributedScheduler");
- return r;
+ public int getBadQueryHistoryNum() {
+ return Integer.parseInt(getOptional("kylin.query.badquery-history-number", "10"));
}
- public Integer getSchedulerType() {
- return Integer.parseInt(getOptional("kylin.job.scheduler.default", "0"));
+ public int getBadQueryDefaultAlertingSeconds() {
+ return Integer.parseInt(getOptional("kylin.query.badquery-alerting-seconds", "90"));
}
- private Map<Integer, String> convertKeyToInteger(Map<String, String> map) {
- Map<Integer, String> result = Maps.newLinkedHashMap();
- for (Entry<String, String> entry : map.entrySet()) {
- result.put(Integer.valueOf(entry.getKey()), entry.getValue());
- }
- return result;
+ public int getBadQueryDefaultDetectIntervalSeconds() {
+ return Integer.parseInt(getOptional("kylin.query.badquery-detect-interval", "60"));
}
- public boolean getCompressionResult() {
- return Boolean.parseBoolean(getOptional("kylin.storage.hbase.endpoint-compress-result", "true"));
+ public boolean getBadQueryPersistentEnabled() {
+ return Boolean.parseBoolean(getOptional("kylin.query.badquery-persistent-enabled", "true"));
+ }
+
+ public String[] getQueryTransformers() {
+ return getOptionalStringArray("kylin.query.transformers", new String[0]);
+ }
+
+ public long getQueryDurationCacheThreshold() {
+ return Long.parseLong(this.getOptional("kylin.query.cache-threshold-duration", String.valueOf(2000)));
+ }
+
+ public long getQueryScanCountCacheThreshold() {
+ return Long.parseLong(this.getOptional("kylin.query.cache-threshold-scan-count", String.valueOf(10 * 1024)));
+ }
+
+ public long getQueryMemBudget() {
+ return Long.parseLong(this.getOptional("kylin.query.memory-budget-bytes", String.valueOf(3L * 1024 * 1024 * 1024)));
+ }
+
+ public boolean isQuerySecureEnabled() {
+ return Boolean.parseBoolean(this.getOptional("kylin.query.security-enabled", "true"));
+ }
+
+ public boolean isQueryCacheEnabled() {
+ return Boolean.parseBoolean(this.getOptional("kylin.query.cache-enabled", "true"));
+ }
+
+ public boolean isQueryIgnoreUnknownFunction() {
+ return Boolean.parseBoolean(this.getOptional("kylin.query.ignore-unknown-function", "false"));
+ }
+
+ public String getQueryAccessController() {
+ return getOptional("kylin.query.access-controller", null);
+ }
+
+ public int getDimCountDistinctMaxCardinality() {
+ return Integer.parseInt(getOptional("kylin.query.max-dimension-count-distinct", "5000000"));
}
public Map<String, String> getUDFs() {
@@ -806,46 +842,54 @@ abstract public class KylinConfigBase implements Serializable {
return udfMap;
}
- public int getHBaseMaxConnectionThreads() {
- return Integer.parseInt(getOptional("kylin.storage.hbase.max-hconnection-threads", "2048"));
+ // ============================================================================
+ // SERVER
+ // ============================================================================
+
+ public String getServerMode() {
+ return this.getOptional("kylin.server.mode", "all");
}
- public int getHBaseCoreConnectionThreads() {
- return Integer.parseInt(getOptional("kylin.storage.hbase.core-hconnection-threads", "2048"));
+ public String[] getRestServers() {
+ return getOptionalStringArray("kylin.server.cluster-servers", new String[0]);
}
- public long getHBaseConnectionThreadPoolAliveSeconds() {
- return Long.parseLong(getOptional("kylin.storage.hbase.hconnection-threads-alive-seconds", "60"));
+ public String getClusterName() {
+ return this.getOptional("kylin.server.cluster-name", getMetadataUrlPrefix());
}
- public int getAppendDictEntrySize() {
- return Integer.parseInt(getOptional("kylin.dictionary.append-entry-size", "10000000"));
+ public String getInitTasks() {
+ return getOptional("kylin.server.init-tasks");
}
- // for test
- public void setAppendDictEntrySize(int entrySize) {
- setProperty("kylin.dictionary.append-entry-size", String.valueOf(entrySize));
+ public int getWorkersPerServer() {
+ //for sequence sql use
+ return Integer.parseInt(getOptional("kylin.server.sequence-sql.workers-per-server", "1"));
}
- public int getAppendDictCacheSize() {
- return Integer.parseInt(getOptional("kylin.dictionary.append-cache-size", "20"));
+ public long getSequenceExpireTime() {
+ return Long.valueOf(this.getOptional("kylin.server.sequence-sql.expire-time", "86400000"));//default a day
}
- // for test
- public void setAppendDictCacheSize(int cacheSize) {
- setProperty("kylin.dictionary.append-cache-size", String.valueOf(cacheSize));
+ public boolean getQueryMetricsEnabled() {
+ return Boolean.parseBoolean(getOptional("kylin.server.query-metrics-enabled", "false"));
}
- @Deprecated
- public String getCreateFlatHiveTableMethod() {
- return getOptional("kylin.source.hive.create-flat-table-method", "1");
+ public int[] getQueryMetricsPercentilesIntervals() {
+ String[] dft = { "60", "300", "3600" };
+ return getOptionalIntArray("kylin.server.query-metrics-percentiles-intervals", dft);
}
- public int getMaxBuildingSegments() {
- return Integer.parseInt(getOptional("kylin.cube.max-building-segments", "10"));
+ // ============================================================================
+ // WEB
+ // ============================================================================
+
+ public String getTimeZone() {
+ return getOptional("kylin.web.timezone", "PST");
}
- public void setMaxBuildingSegments(int maxBuildingSegments) {
- setProperty("kylin.cube.max-building-segments", String.valueOf(maxBuildingSegments));
+ public boolean isWebCrossDomainEnabled() {
+ return Boolean.parseBoolean(getOptional("kylin.web.cross-domain-enabled", "true"));
}
+
}