You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/11/26 05:03:59 UTC
[1/3] kylin git commit: KYLIN-2220 Enforce same name between Cube &
CubeDesc
Repository: kylin
Updated Branches:
refs/heads/master ddc003c86 -> 5edffa51d
KYLIN-2220 Enforce same name between Cube & CubeDesc
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/25f0819f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/25f0819f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/25f0819f
Branch: refs/heads/master
Commit: 25f0819fab211749ac81dff4bbd4812e2be637ed
Parents: ddc003c
Author: kangkaisen <ka...@live.com>
Authored: Tue Nov 22 21:26:32 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Nov 26 13:02:16 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/cube/CubeManager.java | 19 +------------------
.../kylin/rest/controller/CubeController.java | 2 +-
.../apache/kylin/rest/service/CubeService.java | 8 --------
3 files changed, 2 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/25f0819f/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 9893040..307bb46 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -617,24 +617,6 @@ public class CubeManager implements IRealizationProvider {
}
}
- private long calculateStartOffsetForAppendSegment(CubeInstance cube) {
- List<CubeSegment> existing = cube.getSegments();
- if (existing.isEmpty()) {
- return 0;
- } else {
- return existing.get(existing.size() - 1).getSourceOffsetEnd();
- }
- }
-
- private long calculateStartDateForAppendSegment(CubeInstance cube) {
- List<CubeSegment> existing = cube.getSegments();
- if (existing.isEmpty()) {
- return cube.getDescriptor().getPartitionDateStart();
- } else {
- return existing.get(existing.size() - 1).getDateRangeEnd();
- }
- }
-
private void checkBuildingSegment(CubeInstance cube) {
int maxBuldingSeg = cube.getConfig().getMaxBuildingSegments();
if (cube.getBuildingSegments().size() >= maxBuldingSeg) {
@@ -923,6 +905,7 @@ public class CubeManager implements IRealizationProvider {
CubeDesc cubeDesc = CubeDescManager.getInstance(config).getCubeDesc(cube.getDescName());
checkNotNull(cubeDesc, "cube descriptor '%s' (for cube '%s') not found", cube.getDescName(), cubeName);
+ checkState(cubeDesc.getName().equals(cubeName), "cube name '%s' must be same as descriptor name '%s', but it is not", cubeName, cubeDesc.getName());
if (!cubeDesc.getError().isEmpty()) {
cube.setStatus(RealizationStatusEnum.DESCBROKEN);
http://git-wip-us.apache.org/repos/asf/kylin/blob/25f0819f/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index e1aa17a..3846d28 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -416,7 +416,7 @@ public class CubeController extends BasicController {
cubeRequest.setMessage("CubeDesc is null.");
return cubeRequest;
}
- String name = CubeService.getCubeNameFromDesc(desc.getName());
+ String name = desc.getName();
if (StringUtils.isEmpty(name)) {
logger.info("Cube name should not be empty.");
throw new BadRequestException("Cube name should not be empty.");
http://git-wip-us.apache.org/repos/asf/kylin/blob/25f0819f/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 5c59e1a..85c9284 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -278,14 +278,6 @@ public class CubeService extends BasicService {
accessService.clean(cube, true);
}
- public static String getCubeNameFromDesc(String descName) {
- if (descName.toLowerCase().endsWith(DESC_SUFFIX)) {
- return descName.substring(0, descName.toLowerCase().indexOf(DESC_SUFFIX));
- } else {
- return descName;
- }
- }
-
/**
* Stop all jobs belonging to this cube and clean out all segments
*
[2/3] kylin git commit: KYLIN-2217 add saveDictionary() on
CubeManager & DictionaryManager
Posted by li...@apache.org.
KYLIN-2217 add saveDictionary() on CubeManager & DictionaryManager
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ff2d79c4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ff2d79c4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ff2d79c4
Branch: refs/heads/master
Commit: ff2d79c4e032ef243e584730738e5ba2d12ed5c0
Parents: 25f0819
Author: Li Yang <li...@apache.org>
Authored: Thu Nov 24 17:24:57 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Nov 26 13:02:17 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/cube/CubeManager.java | 29 ++++++---
.../kylin/cube/cli/DictionaryGeneratorCLI.java | 34 +++++++++-
.../org/apache/kylin/cube/util/CubingUtils.java | 2 +-
.../org/apache/kylin/dict/DictionaryInfo.java | 5 ++
.../apache/kylin/dict/DictionaryManager.java | 65 +++++++++-----------
.../engine/mr/steps/MergeCuboidMapperTest.java | 2 +-
.../kylin/cube/ITDictionaryManagerTest.java | 6 +-
7 files changed, 93 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ff2d79c4/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 307bb46..ea8ff81 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -48,7 +48,6 @@ import org.apache.kylin.cube.model.DictionaryDesc;
import org.apache.kylin.cube.model.DimensionDesc;
import org.apache.kylin.dict.DictionaryInfo;
import org.apache.kylin.dict.DictionaryManager;
-import org.apache.kylin.dict.DistinctColumnValuesProvider;
import org.apache.kylin.dict.lookup.LookupStringTable;
import org.apache.kylin.dict.lookup.SnapshotManager;
import org.apache.kylin.dict.lookup.SnapshotTable;
@@ -214,25 +213,39 @@ public class CubeManager implements IRealizationProvider {
return result;
}
- public DictionaryInfo buildDictionary(CubeSegment cubeSeg, TblColRef col, DistinctColumnValuesProvider factTableValueProvider) throws IOException {
+ public DictionaryInfo buildDictionary(CubeSegment cubeSeg, TblColRef col, ReadableTable inpTable) throws IOException {
CubeDesc cubeDesc = cubeSeg.getCubeDesc();
if (!cubeDesc.getAllColumnsNeedDictionaryBuilt().contains(col))
return null;
- DictionaryManager dictMgr = getDictionaryManager();
String builderClass = cubeDesc.getDictionaryBuilderClass(col);
- DictionaryInfo dictInfo = dictMgr.buildDictionary(cubeDesc.getModel(), col, factTableValueProvider, builderClass);
+ DictionaryInfo dictInfo = getDictionaryManager().buildDictionary(cubeDesc.getModel(), col, inpTable, builderClass);
+ saveDictionaryInfo(cubeSeg, col, dictInfo);
+ return dictInfo;
+ }
+
+ public DictionaryInfo saveDictionary(CubeSegment cubeSeg, TblColRef col, ReadableTable inpTable, Dictionary<String> dict) throws IOException {
+ CubeDesc cubeDesc = cubeSeg.getCubeDesc();
+ if (!cubeDesc.getAllColumnsNeedDictionaryBuilt().contains(col))
+ return null;
+
+ DictionaryInfo dictInfo = getDictionaryManager().saveDictionary(cubeDesc.getModel(), col, inpTable, dict);
+
+ saveDictionaryInfo(cubeSeg, col, dictInfo);
+ return dictInfo;
+ }
+
+ private void saveDictionaryInfo(CubeSegment cubeSeg, TblColRef col, DictionaryInfo dictInfo) throws IOException {
if (dictInfo != null) {
Dictionary<?> dict = dictInfo.getDictionaryObject();
cubeSeg.putDictResPath(col, dictInfo.getResourcePath());
cubeSeg.getRowkeyStats().add(new Object[] { col.getName(), dict.getSize(), dict.getSizeOfId() });
- CubeUpdate cubeBuilder = new CubeUpdate(cubeSeg.getCubeInstance());
- cubeBuilder.setToUpdateSegs(cubeSeg);
- updateCube(cubeBuilder);
+ CubeUpdate update = new CubeUpdate(cubeSeg.getCubeInstance());
+ update.setToUpdateSegs(cubeSeg);
+ updateCube(update);
}
- return dictInfo;
}
/**
http://git-wip-us.apache.org/repos/asf/kylin/blob/ff2d79c4/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
index 89e2e9b..a6aeb96 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
@@ -26,9 +26,15 @@ import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.model.DimensionDesc;
+import org.apache.kylin.dict.DictionaryManager;
import org.apache.kylin.dict.DistinctColumnValuesProvider;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.source.ReadableTable;
+import org.apache.kylin.source.SourceFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -51,7 +57,8 @@ public class DictionaryGeneratorCLI {
// dictionary
for (TblColRef col : cubeSeg.getCubeDesc().getAllColumnsNeedDictionaryBuilt()) {
logger.info("Building dictionary for " + col);
- cubeMgr.buildDictionary(cubeSeg, col, factTableValueProvider);
+ ReadableTable inpTable = decideInputTable(cubeSeg.getModel(), col, factTableValueProvider);
+ cubeMgr.buildDictionary(cubeSeg, col, inpTable);
}
// snapshot
@@ -67,4 +74,29 @@ public class DictionaryGeneratorCLI {
cubeMgr.buildSnapshotTable(cubeSeg, tableIdentity);
}
}
+
+ private static ReadableTable decideInputTable(DataModelDesc model, TblColRef col, DistinctColumnValuesProvider factTableValueProvider) {
+ KylinConfig config = model.getConfig();
+ DictionaryManager dictMgr = DictionaryManager.getInstance(config);
+ TblColRef srcCol = dictMgr.decideSourceData(model, col);
+ String srcTable = srcCol.getTable();
+
+ ReadableTable inpTable;
+ if (model.isFactTable(srcTable)) {
+ inpTable = factTableValueProvider.getDistinctValuesFor(srcCol);
+ } else {
+ MetadataManager metadataManager = MetadataManager.getInstance(config);
+ TableDesc tableDesc = new TableDesc(metadataManager.getTableDesc(srcTable));
+ if (TableDesc.TABLE_TYPE_VIRTUAL_VIEW.equalsIgnoreCase(tableDesc.getTableType())) {
+ TableDesc materializedTbl = new TableDesc();
+ materializedTbl.setDatabase(config.getHiveDatabaseForIntermediateTable());
+ materializedTbl.setName(tableDesc.getMaterializedName());
+ inpTable = SourceFactory.createReadableTable(materializedTbl);
+ } else {
+ inpTable = SourceFactory.createReadableTable(tableDesc);
+ }
+ }
+
+ return inpTable;
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ff2d79c4/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
index 10dad2c..413b907 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
@@ -170,7 +170,7 @@ public class CubingUtils {
signature.setLastModifiedTime(System.currentTimeMillis());
signature.setPath(String.format("streaming_%s_%s", startOffset, endOffset));
signature.setSize(endOffset - startOffset);
- DictionaryInfo dictInfo = new DictionaryInfo(tblColRef.getTable(), tblColRef.getName(), tblColRef.getColumnDesc().getZeroBasedIndex(), tblColRef.getDatatype(), signature);
+ DictionaryInfo dictInfo = new DictionaryInfo(tblColRef.getColumnDesc(), tblColRef.getDatatype(), signature);
logger.info("writing dictionary for TblColRef:" + tblColRef.toString());
DictionaryManager dictionaryManager = DictionaryManager.getInstance(cubeSegment.getCubeDesc().getConfig());
try {
http://git-wip-us.apache.org/repos/asf/kylin/blob/ff2d79c4/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
index 4fba59a..8526467 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
@@ -21,6 +21,7 @@ package org.apache.kylin.dict;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.source.ReadableTable.TableSignature;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
@@ -51,6 +52,10 @@ public class DictionaryInfo extends RootPersistentEntity {
public DictionaryInfo() {
}
+ public DictionaryInfo(ColumnDesc col, String dataType, TableSignature input) {
+ this(col.getTable().getIdentity(), col.getName(), col.getZeroBasedIndex(), dataType, input);
+ }
+
public DictionaryInfo(String sourceTable, String sourceColumn, int sourceColumnIndex, String dataType, TableSignature input) {
this.updateRandomUuid();
http://git-wip-us.apache.org/repos/asf/kylin/blob/ff2d79c4/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index c33cd28..37e4757 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -38,12 +38,10 @@ import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.JoinDesc;
-import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.source.ReadableTable;
import org.apache.kylin.source.ReadableTable.TableSignature;
-import org.apache.kylin.source.SourceFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -273,45 +271,19 @@ public class DictionaryManager {
}
}
- public DictionaryInfo buildDictionary(DataModelDesc model, TblColRef col, DistinctColumnValuesProvider factTableValueProvider) throws IOException {
- return buildDictionary(model, col, factTableValueProvider, null);
+ public DictionaryInfo buildDictionary(DataModelDesc model, TblColRef col, ReadableTable inpTable) throws IOException {
+ return buildDictionary(model, col, inpTable, null);
}
- public DictionaryInfo buildDictionary(DataModelDesc model, TblColRef col, DistinctColumnValuesProvider factTableValueProvider, String builderClass) throws IOException {
+ public DictionaryInfo buildDictionary(DataModelDesc model, TblColRef col, ReadableTable inpTable, String builderClass) throws IOException {
logger.info("building dictionary for " + col);
- TblColRef srcCol = decideSourceData(model, col);
- String srcTable = srcCol.getTable();
- String srcColName = srcCol.getName();
- int srcColIdx = srcCol.getColumnDesc().getZeroBasedIndex();
-
- ReadableTable inpTable;
- if (model.isFactTable(srcTable)) {
- inpTable = factTableValueProvider.getDistinctValuesFor(srcCol);
- } else {
- MetadataManager metadataManager = MetadataManager.getInstance(config);
- TableDesc tableDesc = new TableDesc(metadataManager.getTableDesc(srcTable));
- if (TableDesc.TABLE_TYPE_VIRTUAL_VIEW.equalsIgnoreCase(tableDesc.getTableType())) {
- TableDesc materializedTbl = new TableDesc();
- materializedTbl.setDatabase(config.getHiveDatabaseForIntermediateTable());
- materializedTbl.setName(tableDesc.getMaterializedName());
- inpTable = SourceFactory.createReadableTable(materializedTbl);
- } else {
- inpTable = SourceFactory.createReadableTable(tableDesc);
- }
- }
-
- TableSignature inputSig = inpTable.getSignature();
- if (inputSig == null) // table does not exists
- return null;
-
- DictionaryInfo dictInfo = new DictionaryInfo(srcTable, srcColName, srcColIdx, col.getDatatype(), inputSig);
-
- String dupDict = checkDupByInfo(dictInfo);
- if (dupDict != null) {
- logger.info("Identical dictionary input " + dictInfo.getInput() + ", reuse existing dictionary at " + dupDict);
- return getDictionaryInfo(dupDict);
+ DictionaryInfo dictInfo = createDictionaryInfo(model, col, inpTable);
+ String dupInfo = checkDupByInfo(dictInfo);
+ if (dupInfo != null) {
+ logger.info("Identical dictionary input " + dictInfo.getInput() + ", reuse existing dictionary at " + dupInfo);
+ return getDictionaryInfo(dupInfo);
}
logger.info("Building dictionary object " + JsonUtil.writeValueAsString(dictInfo));
@@ -333,6 +305,27 @@ public class DictionaryManager {
return trySaveNewDict(dictionary, dictInfo);
}
+ public DictionaryInfo saveDictionary(DataModelDesc model, TblColRef col, ReadableTable inpTable, Dictionary<String> dictionary) throws IOException {
+ DictionaryInfo dictInfo = createDictionaryInfo(model, col, inpTable);
+ String dupInfo = checkDupByInfo(dictInfo);
+ if (dupInfo != null) {
+ logger.info("Identical dictionary input " + dictInfo.getInput() + ", reuse existing dictionary at " + dupInfo);
+ return getDictionaryInfo(dupInfo);
+ }
+
+ return trySaveNewDict(dictionary, dictInfo);
+ }
+
+ private DictionaryInfo createDictionaryInfo(DataModelDesc model, TblColRef col, ReadableTable inpTable) throws IOException {
+ TblColRef srcCol = decideSourceData(model, col);
+ TableSignature inputSig = inpTable.getSignature();
+ if (inputSig == null) // table does not exists
+ throw new IllegalStateException("Input table does not exist: " + inpTable);
+
+ DictionaryInfo dictInfo = new DictionaryInfo(srcCol.getColumnDesc(), col.getDatatype(), inputSig);
+ return dictInfo;
+ }
+
/**
* Decide a dictionary's source data, leverage PK-FK relationship.
*/
http://git-wip-us.apache.org/repos/asf/kylin/blob/ff2d79c4/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java
index 6f3b7c9..075ec80 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java
@@ -119,7 +119,7 @@ public class MergeCuboidMapperTest extends LocalFileMetadataTestCase {
signature.setLastModifiedTime(System.currentTimeMillis());
signature.setPath("fake_dict_for" + lfn.getName() + segment.getName());
- DictionaryInfo newDictInfo = new DictionaryInfo(lfn.getTable(), lfn.getColumnDesc().getName(), lfn.getColumnDesc().getZeroBasedIndex(), "string", signature);
+ DictionaryInfo newDictInfo = new DictionaryInfo(lfn.getColumnDesc(), "string", signature);
List<String> values = new ArrayList<>();
values.add("aaa");
http://git-wip-us.apache.org/repos/asf/kylin/blob/ff2d79c4/kylin-it/src/test/java/org/apache/kylin/cube/ITDictionaryManagerTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/cube/ITDictionaryManagerTest.java b/kylin-it/src/test/java/org/apache/kylin/cube/ITDictionaryManagerTest.java
index 22ffd26..188a97a 100644
--- a/kylin-it/src/test/java/org/apache/kylin/cube/ITDictionaryManagerTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/cube/ITDictionaryManagerTest.java
@@ -65,10 +65,10 @@ public class ITDictionaryManagerTest extends LocalFileMetadataTestCase {
MockDistinctColumnValuesProvider mockupData = new MockDistinctColumnValuesProvider("A", "B", "C");
- DictionaryInfo info1 = dictMgr.buildDictionary(cubeDesc.getModel(), col, mockupData);
+ DictionaryInfo info1 = dictMgr.buildDictionary(cubeDesc.getModel(), col, mockupData.getDistinctValuesFor(col));
System.out.println(JsonUtil.writeValueAsIndentString(info1));
- DictionaryInfo info2 = dictMgr.buildDictionary(cubeDesc.getModel(), col, mockupData);
+ DictionaryInfo info2 = dictMgr.buildDictionary(cubeDesc.getModel(), col, mockupData.getDistinctValuesFor(col));
System.out.println(JsonUtil.writeValueAsIndentString(info2));
// test check duplicate
@@ -89,7 +89,7 @@ public class ITDictionaryManagerTest extends LocalFileMetadataTestCase {
// test empty dictionary
MockDistinctColumnValuesProvider mockupEmpty = new MockDistinctColumnValuesProvider();
- DictionaryInfo info3 = dictMgr.buildDictionary(cubeDesc.getModel(), col, mockupEmpty);
+ DictionaryInfo info3 = dictMgr.buildDictionary(cubeDesc.getModel(), col, mockupEmpty.getDistinctValuesFor(col));
System.out.println(JsonUtil.writeValueAsIndentString(info3));
assertEquals(0, info3.getCardinality());
assertEquals(0, info3.getDictionaryObject().getSize());
[3/3] kylin git commit: KYLIN-2217 add ReadableTable.exists() and
unit tests
Posted by li...@apache.org.
KYLIN-2217 add ReadableTable.exists() and unit tests
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5edffa51
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5edffa51
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5edffa51
Branch: refs/heads/master
Commit: 5edffa51df0cfd6c34682a0c999aac2a114592d2
Parents: ff2d79c
Author: Yang Li <li...@apache.org>
Authored: Fri Nov 25 07:16:45 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Nov 26 13:02:22 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConfig.java | 3 +-
.../apache/kylin/common/KylinConfigBase.java | 8 +-
.../kylin-backward-compatibility.properties | 3 +-
.../java/org/apache/kylin/cube/CubeManager.java | 8 +-
.../apache/kylin/dict/DictionaryManager.java | 4 +-
.../dict/IterableDictionaryValueEnumerator.java | 2 +-
.../apache/kylin/dict/lookup/SnapshotTable.java | 5 ++
.../kylin/dict/DictionaryManagerTest.java | 36 ++++++++
.../apache/kylin/dict/MockupReadableTable.java | 88 ++++++++++++++++++++
.../kylin/job/engine/JobEngineConfig.java | 1 -
.../org/apache/kylin/source/ReadableTable.java | 4 +-
.../apache/kylin/engine/mr/DFSFileTable.java | 17 +++-
.../kylin/engine/mr/SortedColumnDFSFile.java | 5 ++
kylin-it/pom.xml | 6 +-
pom.xml | 2 +-
.../kylin/rest/service/CacheServiceTest.java | 6 +-
.../org/apache/kylin/source/hive/HiveTable.java | 7 +-
17 files changed, 182 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5edffa51/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index 73e5aeb..d888fd3 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -222,7 +222,8 @@ public class KylinConfig extends KylinConfigBase {
Properties propOverride = new Properties();
propOverride.load(ois);
IOUtils.closeQuietly(ois);
- conf.putAll(propOverride);
+ conf = BCC.check(conf);
+ conf.putAll(BCC.check(propOverride));
}
} catch (IOException e) {
throw new RuntimeException(e);
http://git-wip-us.apache.org/repos/asf/kylin/blob/5edffa51/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 8ea03be..aa5e6e1 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -364,10 +364,6 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.parseInt(getOptional("kylin.job.max-concurrent-jobs", "10"));
}
- public String[] getAdminDls() {
- return getOptionalStringArray("kylin.job.notification-admin-emails", null);
- }
-
public int getCubingInMemSamplingPercent() {
int percent = Integer.parseInt(this.getOptional("kylin.job.sampling-percentage", "100"));
percent = Math.max(percent, 1);
@@ -403,6 +399,10 @@ abstract public class KylinConfigBase implements Serializable {
return getOptional("kylin.job.notification-mail-sender", "");
}
+ public String[] getAdminDls() {
+ return getOptionalStringArray("kylin.job.notification-admin-emails", null);
+ }
+
public int getJobRetry() {
return Integer.parseInt(this.getOptional("kylin.job.retry", "0"));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5edffa51/core-common/src/main/resources/kylin-backward-compatibility.properties
----------------------------------------------------------------------
diff --git a/core-common/src/main/resources/kylin-backward-compatibility.properties b/core-common/src/main/resources/kylin-backward-compatibility.properties
index 2ad479d..16871d8 100644
--- a/core-common/src/main/resources/kylin-backward-compatibility.properties
+++ b/core-common/src/main/resources/kylin-backward-compatibility.properties
@@ -40,7 +40,6 @@ kylin.cube.building.segment.max=kylin.cube.max-building-segments
### JOB ###
kylin.job.log.dir=kylin.job.log-dir
-kylin.job.remote.cli.working.dir=kylin.job.remote-cli-working-dir
kylin.job.allow.empty.segment=kylin.job.allow-empty-segment
kylin.job.concurrent.max.limit=kylin.job.max-concurrent-jobs
kylin.job.cubing.inmem.sampling.percent=kylin.job.sampling-percentage
@@ -64,6 +63,7 @@ kylin.job.remote.cli.port=kylin.job.remote-cli-port
kylin.job.remote.cli.hostname=kylin.job.remote-cli-hostname
kylin.job.remote.cli.username=kylin.job.remote-cli-username
kylin.job.remote.cli.password=kylin.job.remote-cli-password
+kylin.job.remote.cli.working.dir=kylin.job.remote-cli-working-dir
### ENGINE ###
@@ -209,6 +209,7 @@ kylin.web.help.=kylin.web.help.
#deprecated
kylin.web.hive.limit=kylin.web.hive-limit
+
### TEST ###
kylin.test.bcc.old.key=kylin.test.bcc.new.key
http://git-wip-us.apache.org/repos/asf/kylin/blob/5edffa51/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index ea8ff81..b4422d2 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -918,7 +918,8 @@ public class CubeManager implements IRealizationProvider {
CubeDesc cubeDesc = CubeDescManager.getInstance(config).getCubeDesc(cube.getDescName());
checkNotNull(cubeDesc, "cube descriptor '%s' (for cube '%s') not found", cube.getDescName(), cubeName);
- checkState(cubeDesc.getName().equals(cubeName), "cube name '%s' must be same as descriptor name '%s', but it is not", cubeName, cubeDesc.getName());
+ if (!isITTestCube(cubeName))
+ checkState(cubeDesc.getName().equals(cubeName), "cube name '%s' must be same as descriptor name '%s', but it is not", cubeName, cubeDesc.getName());
if (!cubeDesc.getError().isEmpty()) {
cube.setStatus(RealizationStatusEnum.DESCBROKEN);
@@ -947,6 +948,11 @@ public class CubeManager implements IRealizationProvider {
}
}
+ private boolean isITTestCube(String cubeName) {
+ return config.isDevEnv() //
+ && (cubeName.startsWith("test_kylin_cube") || cubeName.startsWith("test_streaming"));
+ }
+
private MetadataManager getMetadataManager() {
return MetadataManager.getInstance(config);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5edffa51/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index 37e4757..820299c 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -148,6 +148,7 @@ public class DictionaryManager {
initDictInfo(newDict, newDictInfo);
if (config.isGrowingDictEnabled()) {
+ logger.info("Growing dict is enabled, merge with largest dictionary");
DictionaryInfo largestDictInfo = findLargestDictInfo(newDictInfo);
if (largestDictInfo != null) {
largestDictInfo = getDictionaryInfo(largestDictInfo.getResourcePath());
@@ -167,7 +168,6 @@ public class DictionaryManager {
return saveNewDict(newDictInfo);
}
} else {
- logger.info("Growing dict is not enabled");
String dupDict = checkDupByContent(newDictInfo, newDict);
if (dupDict != null) {
logger.info("Identical dictionary content, reuse existing dictionary at " + dupDict);
@@ -276,6 +276,8 @@ public class DictionaryManager {
}
public DictionaryInfo buildDictionary(DataModelDesc model, TblColRef col, ReadableTable inpTable, String builderClass) throws IOException {
+ if (inpTable.exists() == false)
+ return null;
logger.info("building dictionary for " + col);
http://git-wip-us.apache.org/repos/asf/kylin/blob/5edffa51/core-dictionary/src/main/java/org/apache/kylin/dict/IterableDictionaryValueEnumerator.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/IterableDictionaryValueEnumerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/IterableDictionaryValueEnumerator.java
index 24ebd6d..5c80a26 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/IterableDictionaryValueEnumerator.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/IterableDictionaryValueEnumerator.java
@@ -28,7 +28,7 @@ import java.util.Iterator;
public class IterableDictionaryValueEnumerator implements IDictionaryValueEnumerator {
Iterator<String> iterator;
- public IterableDictionaryValueEnumerator(String[] strs) {
+ public IterableDictionaryValueEnumerator(String... strs) {
this(Arrays.asList(strs));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5edffa51/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
index f2edcb4..9d38dba 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
@@ -164,6 +164,11 @@ public class SnapshotTable extends RootPersistentEntity implements ReadableTable
return signature;
}
+ @Override
+ public boolean exists() throws IOException {
+ return true;
+ }
+
/**
* a naive implementation
*
http://git-wip-us.apache.org/repos/asf/kylin/blob/5edffa51/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryManagerTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryManagerTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryManagerTest.java
index 930d010..e6e0374 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryManagerTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryManagerTest.java
@@ -20,7 +20,10 @@ package org.apache.kylin.dict;
import static org.junit.Assert.*;
+import java.io.IOException;
+
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.DataModelDesc;
@@ -64,4 +67,37 @@ public class DictionaryManagerTest extends LocalFileMetadataTestCase {
assertEquals(factDate, dictMgr.decideSourceData(outerModel, factDate));
}
}
+
+ @Test
+ public void testBuildSaveDictionary() throws IOException {
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ DictionaryManager dictMgr = DictionaryManager.getInstance(config);
+ MetadataManager metaMgr = MetadataManager.getInstance(config);
+ DataModelDesc model = metaMgr.getDataModelDesc("test_kylin_inner_join_model_desc");
+ TblColRef col = model.findColumn("lstg_format_name");
+
+ // non-exist input returns null;
+ DictionaryInfo nullInfo = dictMgr.buildDictionary(model, col, MockupReadableTable.newNonExistTable("/a/path"));
+ assertEquals(null, nullInfo);
+
+ DictionaryInfo info1 = dictMgr.buildDictionary(model, col, MockupReadableTable.newSingleColumnTable("/a/path", "1", "2", "3"));
+ assertEquals(3, info1.getDictionaryObject().getSize());
+
+ // same input returns same dict
+ DictionaryInfo info2 = dictMgr.buildDictionary(model, col, MockupReadableTable.newSingleColumnTable("/a/path", "1", "2", "3"));
+ assertTrue(info1 == info2);
+
+ // same input values (different path) returns same dict
+ DictionaryInfo info3 = dictMgr.buildDictionary(model, col, MockupReadableTable.newSingleColumnTable("/a/different/path", "1", "2", "3"));
+ assertTrue(info1 == info3);
+
+ // save dictionary works in spite of non-exist table
+ Dictionary<String> dict = DictionaryGenerator.buildDictionary(col.getType(), new IterableDictionaryValueEnumerator("1", "2", "3"));
+ DictionaryInfo info4 = dictMgr.saveDictionary(model, col, MockupReadableTable.newNonExistTable("/a/path"), dict);
+ assertTrue(info1 == info4);
+
+ Dictionary<String> dict2 = DictionaryGenerator.buildDictionary(col.getType(), new IterableDictionaryValueEnumerator("1", "2", "3", "4"));
+ DictionaryInfo info5 = dictMgr.saveDictionary(model, col, MockupReadableTable.newNonExistTable("/a/path"), dict2);
+ assertTrue(info1 != info5);
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5edffa51/core-dictionary/src/test/java/org/apache/kylin/dict/MockupReadableTable.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/MockupReadableTable.java b/core-dictionary/src/test/java/org/apache/kylin/dict/MockupReadableTable.java
new file mode 100644
index 0000000..4ee279f
--- /dev/null
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/MockupReadableTable.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.dict;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.kylin.source.ReadableTable;
+
+public class MockupReadableTable implements ReadableTable {
+
+ public static ReadableTable newSingleColumnTable(String path, String... values) {
+ TableSignature sig = new TableSignature(path, values.length, 0);
+ List<String[]> content = new ArrayList<>();
+ for (String v : values) {
+ content.add(new String[] { v });
+ }
+ return new MockupReadableTable(content, sig, true);
+ }
+
+ public static ReadableTable newNonExistTable(String path) {
+ TableSignature sig = new TableSignature(path, -1, 0);
+ return new MockupReadableTable(null, sig, false);
+ }
+
+ private List<String[]> content;
+ private TableSignature sig;
+ private boolean exists;
+
+ public MockupReadableTable(List<String[]> content, TableSignature sig, boolean exists) {
+ this.content = content;
+ this.sig = sig;
+ this.exists = exists;
+ }
+
+ @Override
+ public TableReader getReader() throws IOException {
+ return new TableReader() {
+ int i = -1;
+
+ @Override
+ public boolean next() throws IOException {
+ if (content == null)
+ return false;
+
+ i++;
+ return i < content.size();
+ }
+
+ @Override
+ public String[] getRow() {
+ return content.get(i);
+ }
+
+ @Override
+ public void close() throws IOException {
+ }
+ };
+ }
+
+ @Override
+ public TableSignature getSignature() throws IOException {
+ return sig;
+ }
+
+ @Override
+ public boolean exists() throws IOException {
+ return exists;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5edffa51/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
index 1e3eeaa..0f5b7dd 100644
--- a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
+++ b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
@@ -92,7 +92,6 @@ public class JobEngineConfig {
}
}
}
- logger.info("Chosen job conf is : " + path);
return path;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5edffa51/core-metadata/src/main/java/org/apache/kylin/source/ReadableTable.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/source/ReadableTable.java b/core-metadata/src/main/java/org/apache/kylin/source/ReadableTable.java
index f127821..8ea96de 100644
--- a/core-metadata/src/main/java/org/apache/kylin/source/ReadableTable.java
+++ b/core-metadata/src/main/java/org/apache/kylin/source/ReadableTable.java
@@ -35,9 +35,11 @@ public interface ReadableTable {
public TableReader getReader() throws IOException;
/**
- * Used to detect table modifications mainly. Return null in case table does not exist.
+ * Used to detect table modifications.
*/
public TableSignature getSignature() throws IOException;
+
+ public boolean exists() throws IOException;
public interface TableReader extends Closeable {
http://git-wip-us.apache.org/repos/asf/kylin/blob/5edffa51/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java
index 8c1f6bd..ee932ac 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java
@@ -61,11 +61,22 @@ public class DFSFileTable implements ReadableTable {
@Override
public TableSignature getSignature() throws IOException {
+ Pair<Long, Long> sizeAndLastModified;
try {
- Pair<Long, Long> sizeAndLastModified = getSizeAndLastModified(path);
- return new TableSignature(path, sizeAndLastModified.getFirst(), sizeAndLastModified.getSecond());
+ sizeAndLastModified = getSizeAndLastModified(path);
} catch (FileNotFoundException ex) {
- return null;
+ sizeAndLastModified = Pair.newPair(-1L, 0L);
+ }
+ return new TableSignature(path, sizeAndLastModified.getFirst(), sizeAndLastModified.getSecond());
+ }
+
+ @Override
+ public boolean exists() throws IOException {
+ try {
+ getSizeAndLastModified(path);
+ return true;
+ } catch (FileNotFoundException ex) {
+ return false;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5edffa51/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java
index d0ee3ee..62c309a 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java
@@ -84,6 +84,11 @@ public class SortedColumnDFSFile implements ReadableTable {
public TableSignature getSignature() throws IOException {
return dfsFileTable.getSignature();
}
+
+ @Override
+ public boolean exists() throws IOException {
+ return dfsFileTable.exists();
+ }
private Comparator<String> getComparatorByType(DataType type) {
Comparator<String> comparator;
http://git-wip-us.apache.org/repos/asf/kylin/blob/5edffa51/kylin-it/pom.xml
----------------------------------------------------------------------
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index 148b62b..86f2544 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -276,7 +276,7 @@
</property>
<property>
<name>log4j.configuration</name>
- <value>file:${pom.parent.basedir}/build/conf/kylin-tools-log4j.properties</value>
+ <value>file:${project.basedir}/..//build/conf/kylin-tools-log4j.properties</value>
</property>
</systemProperties>
<argLine>-Xms1G -Xmx2G -XX:PermSize=128M -XX:MaxPermSize=512M</argLine>
@@ -300,7 +300,7 @@
<argument>-DuseSandbox=true</argument>
<argument>-Dhdp.version=${hdp.version}</argument>
<argument>-DfastBuildMode=${fastBuildMode}</argument>
- <argument>-Dlog4j.configuration=file:${pom.parent.basedir}/build/conf/kylin-tools-log4j.properties</argument>
+ <argument>-Dlog4j.configuration=file:${project.basedir}/..//build/conf/kylin-tools-log4j.properties</argument>
<argument>-classpath</argument>
<classpath/>
<argument>org.apache.kylin.provision.BuildCubeWithEngine</argument>
@@ -322,7 +322,7 @@
<argument>-DuseSandbox=true</argument>
<argument>-Dhdp.version=${hdp.version}</argument>
<argument>-DfastBuildMode=${fastBuildMode}</argument>
- <argument>-Dlog4j.configuration=file:${pom.parent.basedir}/build/conf/kylin-tools-log4j.properties</argument>
+ <argument>-Dlog4j.configuration=file:${project.basedir}/..//build/conf/kylin-tools-log4j.properties</argument>
<argument>-classpath</argument>
<classpath/>
<argument>org.apache.kylin.provision.BuildCubeWithStream</argument>
http://git-wip-us.apache.org/repos/asf/kylin/blob/5edffa51/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 0501d0f..51479c8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1097,7 +1097,7 @@
</property>
<property>
<name>log4j.configuration</name>
- <value>file:${pom.parent.basedir}/build/conf/kylin-tools-log4j.properties</value>
+ <value>file:${project.basedir}/../build/conf/kylin-tools-log4j.properties</value>
</property>
</systemProperties>
<argLine>-javaagent:${project.build.testOutputDirectory}/jamm.jar ${argLine}</argLine>
http://git-wip-us.apache.org/repos/asf/kylin/blob/5edffa51/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
index 8ea9cd2..7f7b6f0 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
@@ -203,11 +203,11 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
final Broadcaster broadcaster = Broadcaster.getInstance(configA);
broadcaster.getCounterAndClear();
- getStore().deleteResource("/cube/a_whole_new_cube.json");
+ getStore().deleteResource("/cube/test_kylin_cube_a_new_one.json");
//create cube
- final String cubeName = "a_whole_new_cube";
+ final String cubeName = "test_kylin_cube_a_new_one";
final CubeManager cubeManager = getCubeManager(configA);
final CubeManager cubeManagerB = getCubeManager(configB);
final ProjectManager projectManager = getProjectManager(configA);
@@ -277,7 +277,7 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
assertTrue(cubeDescManager.getCubeDesc(cubeDescName) == null);
assertTrue(cubeDescManagerB.getCubeDesc(cubeDescName) == null);
- getStore().deleteResource("/cube/a_whole_new_cube.json");
+ getStore().deleteResource("/cube/test_kylin_cube_a_new_one.json");
}
private TableDesc createTestTableDesc() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/5edffa51/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTable.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTable.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTable.java
index a63aae7..83e49e9 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTable.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTable.java
@@ -78,8 +78,11 @@ public class HiveTable implements ReadableTable {
throw new IOException(e);
}
}
-
-
+
+ @Override
+ public boolean exists() {
+ return true;
+ }
private String computeHDFSLocation() throws Exception {