You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/01/31 13:01:38 UTC
[11/50] incubator-kylin git commit: KYLIN-540 Data model upgrade for
legacy cube descs: move partition, filter and capacity to DataModelDesc.java,
and also updated the upgrade tool.
KYLIN-540 Data model upgrade for legacy cube descs: move partition, filter and capacity to DataModelDesc.java, and also updated the upgrade tool.
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/a1afffdd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/a1afffdd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/a1afffdd
Branch: refs/heads/inverted-index
Commit: a1afffdd1998ed3beb24989f8727c3eeb94973bd
Parents: 1309182
Author: Shao Feng, Shi <sh...@hotmail.com>
Authored: Wed Jan 28 15:28:24 2015 +0800
Committer: Shao Feng, Shi <sh...@hotmail.com>
Committed: Wed Jan 28 15:28:24 2015 +0800
----------------------------------------------------------------------
.../persistence/RootPersistentEntity.java | 2 +
.../com/kylinolap/cube/CubeDescUpgrader.java | 54 +++++-----
.../java/com/kylinolap/cube/CubeManager.java | 13 ++-
.../java/com/kylinolap/cube/model/CubeDesc.java | 42 +-------
.../kylinolap/cube/model/CubePartitionDesc.java | 100 -------------------
.../kylinolap/cube/model/v1/CubeInstance.java | 1 -
.../com/kylinolap/cube/CubeManagerTest.java | 4 +-
.../kylinolap/cube/SegmentManagementTest.java | 14 +--
.../com/kylinolap/invertedindex/IIManager.java | 10 +-
.../com/kylinolap/job/CubeMetadataUpgrade.java | 32 +++++-
.../java/com/kylinolap/job/JoinedFlatTable.java | 6 +-
.../kylinolap/job/cube/CubingJobBuilder.java | 2 +-
.../kylinolap/job/engine/JobEngineConfig.java | 2 +-
.../hadoop/cube/RangeKeyDistributionJob.java | 4 +-
.../hadoop/hive/CubeJoinedFlatTableDesc.java | 4 +-
.../job/hadoop/hive/IIJoinedFlatTableDesc.java | 5 +-
.../job/hadoop/hive/IJoinedFlatTableDesc.java | 7 +-
.../job/invertedindex/IIJobBuilder.java | 4 +-
.../kylinolap/job/BuildCubeWithEngineTest.java | 4 +-
.../job/tools/ColumnCardinalityMapperTest.java | 1 -
.../kylinolap/metadata/model/DataModelDesc.java | 44 +++++++-
.../kylinolap/metadata/model/PartitionDesc.java | 100 +++++++++++++++++++
.../metadata/project/ProjectInstance.java | 11 --
.../com/kylinolap/rest/service/CubeService.java | 5 +-
.../storage/hbase/CubeStorageEngine.java | 2 +-
.../kylinolap/storage/hbase/HBaseKeyRange.java | 2 +-
.../com/kylinolap/storage/MiniClusterTest.java | 7 --
.../storage/minicluster/HiveJDBCClientTest.java | 4 -
.../minicluster/HiveMiniClusterTest.java | 6 --
29 files changed, 242 insertions(+), 250 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a1afffdd/common/src/main/java/com/kylinolap/common/persistence/RootPersistentEntity.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/persistence/RootPersistentEntity.java b/common/src/main/java/com/kylinolap/common/persistence/RootPersistentEntity.java
index 9ee6ec6..a57d4f9 100644
--- a/common/src/main/java/com/kylinolap/common/persistence/RootPersistentEntity.java
+++ b/common/src/main/java/com/kylinolap/common/persistence/RootPersistentEntity.java
@@ -48,6 +48,8 @@ abstract public class RootPersistentEntity implements AclEntity {
}
public static long parseTime(String timeString) {
+ if(timeString == null)
+ return 0;
try {
Date dt = df.parse(timeString);
return dt.getTime();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a1afffdd/cube/src/main/java/com/kylinolap/cube/CubeDescUpgrader.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/CubeDescUpgrader.java b/cube/src/main/java/com/kylinolap/cube/CubeDescUpgrader.java
index d4c2950..6502e5c 100644
--- a/cube/src/main/java/com/kylinolap/cube/CubeDescUpgrader.java
+++ b/cube/src/main/java/com/kylinolap/cube/CubeDescUpgrader.java
@@ -1,7 +1,6 @@
package com.kylinolap.cube;
import java.io.IOException;
-import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -49,7 +48,6 @@ public class CubeDescUpgrader {
DataModelDesc model = extractDataModel(oldModel, newModel);
newModel.setModel(model);
- updatePartitionDesc(oldModel, newModel);
updateDimensions(oldModel, newModel);
@@ -75,20 +73,12 @@ public class CubeDescUpgrader {
newModel.setName(oldModel.getName());
newModel.setDescription(oldModel.getDescription());
newModel.setNullStrings(oldModel.getNullStrings());
- newModel.setFilterCondition(oldModel.getFilterCondition());
newModel.setMeasures(oldModel.getMeasures());
newModel.setRowkey(oldModel.getRowkey());
newModel.setHbaseMapping(oldModel.getHBaseMapping());
newModel.setSignature(oldModel.getSignature());
- if (oldModel.getCapacity() == com.kylinolap.cube.model.v1.CubeDesc.CubeCapacity.SMALL) {
- newModel.setCapacity(com.kylinolap.cube.model.CubeDesc.RealizationCapacity.SMALL);
- } else if (oldModel.getCapacity() == com.kylinolap.cube.model.v1.CubeDesc.CubeCapacity.MEDIUM) {
- newModel.setCapacity(com.kylinolap.cube.model.CubeDesc.RealizationCapacity.MEDIUM);
- } else if (oldModel.getCapacity() == com.kylinolap.cube.model.v1.CubeDesc.CubeCapacity.LARGE) {
- newModel.setCapacity(com.kylinolap.cube.model.CubeDesc.RealizationCapacity.LARGE);
- }
newModel.setNotifyList(oldModel.getNotifyList());
newModel.setLastModified(oldModel.getLastModified());
@@ -114,7 +104,7 @@ public class CubeDescUpgrader {
for (com.kylinolap.cube.model.v1.DimensionDesc dim : oldDimensions) {
com.kylinolap.cube.model.DimensionDesc newDim = null;
- // if a dimension defines "column", "derived" and "hierarchy" at the sametime, separate it into three dimensions;
+ // if a dimension defines "column", "derived" and "hierarchy" at the same time, separate it into three dimensions;
if (dim.getColumn() != null && !"{FK}".equals(dim.getColumn())) {
//column on fact table
@@ -122,7 +112,7 @@ public class CubeDescUpgrader {
newDimensions.add(newDim);
newDim.setColumn(new String[] { dim.getColumn() });
} else if (ArrayUtils.isEmpty(dim.getDerived()) && ArrayUtils.isEmpty(dim.getHierarchy())) {
- // user defines a lookup table, but didn't use any column other than the pk, in this case, covnert to use fact table's fk
+ // user defines a lookup table, but didn't use any column other than the pk, in this case, convert to use fact table's fk
newDim = newDimensionDesc(dim, dimId++, dim.getName());
newDimensions.add(newDim);
newDim.setTable(getMetadataManager().appendDBName(newModel.getFactTable()));
@@ -134,7 +124,7 @@ public class CubeDescUpgrader {
newDim = newDimensionDesc(dim, dimId++, dim.getName() + "_derived");
newDimensions.add(newDim);
newDim.setDerived(dim.getDerived());
- newDim.setColumn(null);
+ newDim.setColumn(null); // derived column must come from a lookup table; in this case the fk will be the dimension column, no need to explicitly declare it;
}
if (!ArrayUtils.isEmpty(dim.getHierarchy())) {
@@ -181,45 +171,55 @@ public class CubeDescUpgrader {
}
dm.setLookups(lookups.toArray(new LookupDesc[lookups.size()]));
+ dm.setFilterCondition(oldModel.getFilterCondition());
+ updatePartitionDesc(oldModel, dm);
+
+
+ if (oldModel.getCapacity() == com.kylinolap.cube.model.v1.CubeDesc.CubeCapacity.SMALL) {
+ dm.setCapacity(com.kylinolap.metadata.model.DataModelDesc.RealizationCapacity.SMALL);
+ } else if (oldModel.getCapacity() == com.kylinolap.cube.model.v1.CubeDesc.CubeCapacity.MEDIUM) {
+ dm.setCapacity(com.kylinolap.metadata.model.DataModelDesc.RealizationCapacity.MEDIUM);
+ } else if (oldModel.getCapacity() == com.kylinolap.cube.model.v1.CubeDesc.CubeCapacity.LARGE) {
+ dm.setCapacity(com.kylinolap.metadata.model.DataModelDesc.RealizationCapacity.LARGE);
+ }
+
return dm;
}
- private void updatePartitionDesc(com.kylinolap.cube.model.v1.CubeDesc oldModel, com.kylinolap.cube.model.CubeDesc newModel) {
+ private void updatePartitionDesc(com.kylinolap.cube.model.v1.CubeDesc oldModel, com.kylinolap.metadata.model.DataModelDesc dm) {
com.kylinolap.cube.model.v1.CubePartitionDesc partition = oldModel.getCubePartitionDesc();
- com.kylinolap.cube.model.CubePartitionDesc newPartition = new com.kylinolap.cube.model.CubePartitionDesc();
- newModel.setCubePartitionDesc(newPartition);
+ com.kylinolap.metadata.model.PartitionDesc newPartition = new com.kylinolap.metadata.model.PartitionDesc();
if (partition.getPartitionDateColumn() != null) {
String partitionCol = partition.getPartitionDateColumn();
-
+
String[] tablecolumn = partitionCol.split("\\.");
if (tablecolumn != null && tablecolumn.length == 2) {
// pattern is <tablename>.<colname>
String tableFullName = getMetadataManager().appendDBName(tablecolumn[0]);
newPartition.setPartitionDateColumn(tableFullName + "." + tablecolumn[1]);
} else {
-
- if(partitionCol.indexOf(".") < 0) {
- // pattern is <colname>
- partitionCol = newModel.getFactTable() + "." + partitionCol;
+
+ if (partitionCol.indexOf(".") < 0) {
+ // pattern is <colname>
+ partitionCol = dm.getFactTable() + "." + partitionCol;
}
-
+
newPartition.setPartitionDateColumn(partitionCol);
}
}
if (partition.getCubePartitionType() == com.kylinolap.cube.model.v1.CubePartitionDesc.CubePartitionType.APPEND) {
- newPartition.setCubePartitionType(com.kylinolap.cube.model.CubePartitionDesc.CubePartitionType.APPEND);
- }
-
- if (partition.getCubePartitionType() == CubePartitionDesc.CubePartitionType.UPDATE_INSERT) {
- newPartition.setCubePartitionType(com.kylinolap.cube.model.CubePartitionDesc.CubePartitionType.UPDATE_INSERT);
+ newPartition.setCubePartitionType(com.kylinolap.metadata.model.PartitionDesc.PartitionType.APPEND);
+ } else if (partition.getCubePartitionType() == CubePartitionDesc.CubePartitionType.UPDATE_INSERT) {
+ newPartition.setCubePartitionType(com.kylinolap.metadata.model.PartitionDesc.PartitionType.UPDATE_INSERT);
}
newPartition.setPartitionDateStart(partition.getPartitionDateStart());
+ dm.setPartitionDesc(newPartition);
}
private com.kylinolap.cube.model.v1.CubeDesc loadOldCubeDesc(String path) throws IOException {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a1afffdd/cube/src/main/java/com/kylinolap/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/CubeManager.java b/cube/src/main/java/com/kylinolap/cube/CubeManager.java
index eb5fe88..e4a3f52 100644
--- a/cube/src/main/java/com/kylinolap/cube/CubeManager.java
+++ b/cube/src/main/java/com/kylinolap/cube/CubeManager.java
@@ -44,9 +44,7 @@ import com.kylinolap.common.restclient.Broadcaster;
import com.kylinolap.common.restclient.CaseInsensitiveStringCache;
import com.kylinolap.cube.model.CubeBuildTypeEnum;
import com.kylinolap.cube.model.CubeDesc;
-import com.kylinolap.cube.model.CubePartitionDesc;
import com.kylinolap.cube.model.DimensionDesc;
-import com.kylinolap.dict.DateStrDictionary;
import com.kylinolap.dict.Dictionary;
import com.kylinolap.dict.DictionaryInfo;
import com.kylinolap.dict.DictionaryManager;
@@ -55,6 +53,7 @@ import com.kylinolap.dict.lookup.LookupStringTable;
import com.kylinolap.dict.lookup.SnapshotManager;
import com.kylinolap.dict.lookup.SnapshotTable;
import com.kylinolap.metadata.MetadataManager;
+import com.kylinolap.metadata.model.PartitionDesc;
import com.kylinolap.metadata.model.SegmentStatusEnum;
import com.kylinolap.metadata.model.TableDesc;
import com.kylinolap.metadata.model.TblColRef;
@@ -259,7 +258,7 @@ public class CubeManager implements IRealizationProvider {
long appendStart = calculateStartDateForAppendSegment(cube);
CubeSegment appendSegment = newSegment(cube, appendStart, endDate);
- long startDate = cube.getDescriptor().getCubePartitionDesc().getPartitionDateStart();
+ long startDate = cube.getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
CubeSegment mergeSegment = newSegment(cube, startDate, endDate);
validateNewSegments(cube, mergeSegment);
@@ -275,7 +274,7 @@ public class CubeManager implements IRealizationProvider {
checkNoBuildingSegment(cube);
CubeSegment newSegment;
- if (cube.getDescriptor().getCubePartitionDesc().isPartitioned()) {
+ if (cube.getDescriptor().getModel().getPartitionDesc().isPartitioned()) {
long startDate = calculateStartDateForAppendSegment(cube);
newSegment = newSegment(cube, startDate, endDate);
} else {
@@ -344,7 +343,7 @@ public class CubeManager implements IRealizationProvider {
private long calculateStartDateForAppendSegment(CubeInstance cube) {
List<CubeSegment> existing = cube.getSegments();
if (existing.isEmpty()) {
- return cube.getDescriptor().getCubePartitionDesc().getPartitionDateStart();
+ return cube.getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
} else {
return existing.get(existing.size() - 1).getDateRangeEnd();
}
@@ -357,7 +356,7 @@ public class CubeManager implements IRealizationProvider {
}
private void checkCubeIsPartitioned(CubeInstance cube) {
- if (cube.getDescriptor().getCubePartitionDesc().isPartitioned() == false) {
+ if (cube.getDescriptor().getModel().getPartitionDesc().isPartitioned() == false) {
throw new IllegalStateException("there is no partition date column specified, only full build is supported");
}
}
@@ -555,7 +554,7 @@ public class CubeManager implements IRealizationProvider {
*/
private List<CubeSegment> calculateToBeSegments(CubeInstance cube, CubeSegment... newSegments) {
CubeDesc cubeDesc = cube.getDescriptor();
- CubePartitionDesc partDesc = cubeDesc.getCubePartitionDesc();
+ PartitionDesc partDesc = cubeDesc.getModel().getPartitionDesc();
List<CubeSegment> tobe = Lists.newArrayList(cube.getSegments());
if (newSegments != null)
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a1afffdd/cube/src/main/java/com/kylinolap/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/model/CubeDesc.java b/cube/src/main/java/com/kylinolap/cube/model/CubeDesc.java
index 5301081..39a4704 100644
--- a/cube/src/main/java/com/kylinolap/cube/model/CubeDesc.java
+++ b/cube/src/main/java/com/kylinolap/cube/model/CubeDesc.java
@@ -66,10 +66,6 @@ import com.kylinolap.metadata.model.TblColRef;
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class CubeDesc extends RootPersistentEntity {
- public static enum RealizationCapacity {
- SMALL, MEDIUM, LARGE
- }
-
public static enum DeriveType {
LOOKUP, PK_FK
}
@@ -105,10 +101,6 @@ public class CubeDesc extends RootPersistentEntity {
private String description;
@JsonProperty("null_string")
private String[] nullStrings;
- @JsonProperty("filter_condition")
- private String filterCondition;
- @JsonProperty("cube_partition_desc")
- CubePartitionDesc cubePartitionDesc;
@JsonProperty("dimensions")
private List<DimensionDesc> dimensions;
@JsonProperty("measures")
@@ -119,8 +111,6 @@ public class CubeDesc extends RootPersistentEntity {
private HBaseMappingDesc hbaseMapping;
@JsonProperty("signature")
private String signature;
- @JsonProperty("capacity")
- private RealizationCapacity capacity = RealizationCapacity.MEDIUM;
@JsonProperty("notify_list")
private List<String> notifyList;
@@ -342,22 +332,6 @@ public class CubeDesc extends RootPersistentEntity {
return nullStrings;
}
- public String getFilterCondition() {
- return filterCondition;
- }
-
- public void setFilterCondition(String filterCondition) {
- this.filterCondition = filterCondition;
- }
-
- public CubePartitionDesc getCubePartitionDesc() {
- return cubePartitionDesc;
- }
-
- public void setCubePartitionDesc(CubePartitionDesc cubePartitionDesc) {
- this.cubePartitionDesc = cubePartitionDesc;
- }
-
public List<DimensionDesc> getDimensions() {
return dimensions;
}
@@ -390,14 +364,6 @@ public class CubeDesc extends RootPersistentEntity {
this.signature = signature;
}
- public RealizationCapacity getCapacity() {
- return capacity;
- }
-
- public void setCapacity(RealizationCapacity capacity) {
- this.capacity = capacity;
- }
-
public List<String> getNotifyList() {
return notifyList;
}
@@ -433,7 +399,7 @@ public class CubeDesc extends RootPersistentEntity {
@Override
public String toString() {
- return "CubeDesc [name=" + name + ", factTable=" + getFactTable() + ", cubePartitionDesc=" + cubePartitionDesc + ", dimensions=" + dimensions + ", measures=" + measures + ", rowkey=" + rowkey + ", hbaseMapping=" + hbaseMapping + "]";
+ return "CubeDesc [name=" + name + ", factTable=" + getFactTable() + ", dimensions=" + dimensions + ", measures=" + measures + ", rowkey=" + rowkey + ", hbaseMapping=" + hbaseMapping + "]";
}
public String calculateSignature() {
@@ -441,7 +407,7 @@ public class CubeDesc extends RootPersistentEntity {
try {
md = MessageDigest.getInstance("MD5");
StringBuilder sigString = new StringBuilder();
- sigString.append(this.name).append("|").append(this.getFactTable()).append("|").append(JsonUtil.writeValueAsString(this.cubePartitionDesc)).append("|").append(JsonUtil.writeValueAsString(this.dimensions)).append("|").append(JsonUtil.writeValueAsString(this.measures)).append("|").append(JsonUtil.writeValueAsString(this.rowkey)).append("|").append(JsonUtil.writeValueAsString(this.hbaseMapping));
+ sigString.append(this.name).append("|").append(this.getFactTable()).append("|").append(JsonUtil.writeValueAsString(this.model.getPartitionDesc())).append("|").append(JsonUtil.writeValueAsString(this.dimensions)).append("|").append(JsonUtil.writeValueAsString(this.measures)).append("|").append(JsonUtil.writeValueAsString(this.rowkey)).append("|").append(JsonUtil.writeValueAsString(this.hbaseMapping));
byte[] signature = md.digest(sigString.toString().getBytes());
return new String(Base64.encodeBase64(signature));
@@ -517,8 +483,8 @@ public class CubeDesc extends RootPersistentEntity {
initMeasureReferenceToColumnFamily();
- if (null != this.cubePartitionDesc) {
- this.cubePartitionDesc.init(columnMap);
+ if (null != this.model.getPartitionDesc()) {
+ this.model.getPartitionDesc().init(columnMap);
}
// check all dimension columns are presented on rowkey
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a1afffdd/cube/src/main/java/com/kylinolap/cube/model/CubePartitionDesc.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/model/CubePartitionDesc.java b/cube/src/main/java/com/kylinolap/cube/model/CubePartitionDesc.java
deleted file mode 100644
index 56c41f4..0000000
--- a/cube/src/main/java/com/kylinolap/cube/model/CubePartitionDesc.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.cube.model;
-
-import java.util.Map;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.kylinolap.common.util.StringSplitter;
-import com.kylinolap.metadata.model.TblColRef;
-
-/**
- * @author xduo
- *
- */
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class CubePartitionDesc {
-
- public static enum CubePartitionType {
- APPEND, //
- UPDATE_INSERT // not used since 0.7.1
- }
-
- @JsonProperty("partition_date_column")
- private String partitionDateColumn;
- @JsonProperty("partition_date_start")
- private long partitionDateStart = 0L;
- @JsonProperty("cube_partition_type")
- private CubePartitionType cubePartitionType = CubePartitionType.APPEND;
-
- private TblColRef partitionDateColumnRef;
-
- void init(Map<String, Map<String, TblColRef>> columnMap) {
- if (null != partitionDateColumn) {
- partitionDateColumn = partitionDateColumn.toUpperCase();
-
- String[] columns = StringSplitter.split(partitionDateColumn, ".");
-
- if (null != columns && columns.length == 3) {
- String tableName = columns[0].toUpperCase() + "." + columns[1].toUpperCase();
- Map<String, TblColRef> cols = columnMap.get(tableName);
- if (cols != null) {
- partitionDateColumnRef = cols.get(columns[2].toUpperCase());
- } else {
- throw new IllegalStateException("The table '" + tableName + "' provided in 'partition_date_column' doesn't exist.");
- }
- } else {
- throw new IllegalStateException("The 'partition_date_column' format is invalid: " + partitionDateColumn + ", it should be {db}.{table}.{column}.");
- }
- }
- }
-
- public boolean isPartitioned() {
- return partitionDateColumnRef != null;
- }
-
- public String getPartitionDateColumn() {
- return partitionDateColumn;
- }
-
- public void setPartitionDateColumn(String partitionDateColumn) {
- this.partitionDateColumn = partitionDateColumn;
- }
-
- public long getPartitionDateStart() {
- return partitionDateStart;
- }
-
- public void setPartitionDateStart(long partitionDateStart) {
- this.partitionDateStart = partitionDateStart;
- }
-
- public CubePartitionType getCubePartitionType() {
- return cubePartitionType;
- }
-
- public void setCubePartitionType(CubePartitionType cubePartitionType) {
- this.cubePartitionType = cubePartitionType;
- }
-
- public TblColRef getPartitionDateColumnRef() {
- return partitionDateColumnRef;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a1afffdd/cube/src/main/java/com/kylinolap/cube/model/v1/CubeInstance.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/model/v1/CubeInstance.java b/cube/src/main/java/com/kylinolap/cube/model/v1/CubeInstance.java
index 6d50c1b..a803169 100644
--- a/cube/src/main/java/com/kylinolap/cube/model/v1/CubeInstance.java
+++ b/cube/src/main/java/com/kylinolap/cube/model/v1/CubeInstance.java
@@ -29,7 +29,6 @@ import com.google.common.collect.Lists;
import com.kylinolap.common.KylinConfig;
import com.kylinolap.common.persistence.ResourceStore;
import com.kylinolap.common.persistence.RootPersistentEntity;
-import com.kylinolap.cube.CubeDescManager;
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class CubeInstance extends RootPersistentEntity {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a1afffdd/cube/src/test/java/com/kylinolap/cube/CubeManagerTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/com/kylinolap/cube/CubeManagerTest.java b/cube/src/test/java/com/kylinolap/cube/CubeManagerTest.java
index 5b1c86d..8339dfa 100644
--- a/cube/src/test/java/com/kylinolap/cube/CubeManagerTest.java
+++ b/cube/src/test/java/com/kylinolap/cube/CubeManagerTest.java
@@ -52,12 +52,12 @@ public class CubeManagerTest extends LocalFileMetadataTestCase {
@Test
public void testBasics() throws Exception {
- CubeInstance cube = CubeManager.getInstance(this.getTestConfig()).getCube("test_kylin_cube_without_slr_ready");
+ CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_without_slr_ready");
CubeDesc desc = cube.getDescriptor();
System.out.println(JsonUtil.writeValueAsIndentString(desc));
String signature = desc.calculateSignature();
- desc.getCubePartitionDesc().setPartitionDateColumn("test_column");
+ desc.getModel().getPartitionDesc().setPartitionDateColumn("test_column");
assertTrue(!signature.equals(desc.calculateSignature()));
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a1afffdd/cube/src/test/java/com/kylinolap/cube/SegmentManagementTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/com/kylinolap/cube/SegmentManagementTest.java b/cube/src/test/java/com/kylinolap/cube/SegmentManagementTest.java
index 3c70aca..13c7ed3 100644
--- a/cube/src/test/java/com/kylinolap/cube/SegmentManagementTest.java
+++ b/cube/src/test/java/com/kylinolap/cube/SegmentManagementTest.java
@@ -101,7 +101,7 @@ public class SegmentManagementTest extends LocalFileMetadataTestCase {
assertEquals(1, cubeInstance.getBuildingSegments().size());
assertEquals(0, cubeInstance.getRebuildingSegments().size());
assertTrue("".equals(initialSegment.getStorageLocationIdentifier()) == false);
- assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
+ assertEquals(desc.getModel().getPartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
assertEquals(dateEnd, cubeInstance.getAllocatedEndDate());
// initial build success
@@ -112,7 +112,7 @@ public class SegmentManagementTest extends LocalFileMetadataTestCase {
assertEquals(1, cubeInstance.getSegments(SegmentStatusEnum.READY).size());
assertEquals(0, cubeInstance.getBuildingSegments().size());
assertEquals(0, cubeInstance.getRebuildingSegments().size());
- assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
+ assertEquals(desc.getModel().getPartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
assertEquals(dateEnd, cubeInstance.getAllocatedEndDate());
// incremental build
@@ -124,7 +124,7 @@ public class SegmentManagementTest extends LocalFileMetadataTestCase {
assertEquals(2, cubeInstance.getSegments().size());
assertEquals(1, cubeInstance.getSegments(SegmentStatusEnum.NEW).size());
assertEquals(1, cubeInstance.getBuildingSegments().size());
- assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
+ assertEquals(desc.getModel().getPartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
assertEquals(dateEnd2, cubeInstance.getAllocatedEndDate());
assertEquals(dateEnd, cubeInstance.getBuildingSegments().get(0).getDateRangeStart());
assertEquals(dateEnd2, cubeInstance.getBuildingSegments().get(0).getDateRangeEnd());
@@ -137,7 +137,7 @@ public class SegmentManagementTest extends LocalFileMetadataTestCase {
assertEquals(2, cubeInstance.getSegments().size());
assertEquals(2, cubeInstance.getSegments(SegmentStatusEnum.READY).size());
assertEquals(0, cubeInstance.getBuildingSegments().size());
- assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
+ assertEquals(desc.getModel().getPartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
assertEquals(dateEnd2, cubeInstance.getAllocatedEndDate());
}
@@ -242,7 +242,7 @@ public class SegmentManagementTest extends LocalFileMetadataTestCase {
assertEquals(0, cubeInstance.getRebuildingSegments().size());
assertTrue("".equals(initialSegment.getStorageLocationIdentifier()) == false);
assertEquals("FULL_BUILD", initialSegment.getName());
- assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
+ assertEquals(desc.getModel().getPartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
assertEquals(0, cubeInstance.getAllocatedEndDate());
// initial build success
@@ -311,7 +311,7 @@ public class SegmentManagementTest extends LocalFileMetadataTestCase {
assertEquals(1, cubeInstance.getBuildingSegments().size());
assertEquals(0, cubeInstance.getRebuildingSegments().size());
assertTrue("".equals(initialSegment.getStorageLocationIdentifier()) == false);
- assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
+ assertEquals(desc.getModel().getPartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
assertEquals(dateEnd, cubeInstance.getAllocatedEndDate());
// initial build success
@@ -322,7 +322,7 @@ public class SegmentManagementTest extends LocalFileMetadataTestCase {
assertEquals(1, cubeInstance.getSegments(SegmentStatusEnum.READY).size());
assertEquals(0, cubeInstance.getBuildingSegments().size());
assertEquals(0, cubeInstance.getRebuildingSegments().size());
- assertEquals(desc.getCubePartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
+ assertEquals(desc.getModel().getPartitionDesc().getPartitionDateStart(), cubeInstance.getAllocatedStartDate());
assertEquals(dateEnd, cubeInstance.getAllocatedEndDate());
// incremental build
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a1afffdd/invertedindex/src/main/java/com/kylinolap/invertedindex/IIManager.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/com/kylinolap/invertedindex/IIManager.java b/invertedindex/src/main/java/com/kylinolap/invertedindex/IIManager.java
index 2965a92..34f4418 100644
--- a/invertedindex/src/main/java/com/kylinolap/invertedindex/IIManager.java
+++ b/invertedindex/src/main/java/com/kylinolap/invertedindex/IIManager.java
@@ -18,17 +18,12 @@ package com.kylinolap.invertedindex;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
-import com.kylinolap.common.restclient.CaseInsensitiveStringCache;
-import com.kylinolap.metadata.realization.IRealization;
-import com.kylinolap.metadata.realization.IRealizationProvider;
-import com.kylinolap.metadata.realization.RealizationType;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -38,7 +33,7 @@ import com.kylinolap.common.persistence.JsonSerializer;
import com.kylinolap.common.persistence.ResourceStore;
import com.kylinolap.common.persistence.Serializer;
import com.kylinolap.common.restclient.Broadcaster;
-import com.kylinolap.dict.DateStrDictionary;
+import com.kylinolap.common.restclient.CaseInsensitiveStringCache;
import com.kylinolap.dict.Dictionary;
import com.kylinolap.dict.DictionaryInfo;
import com.kylinolap.dict.DictionaryManager;
@@ -47,6 +42,9 @@ import com.kylinolap.invertedindex.model.IIDesc;
import com.kylinolap.metadata.MetadataManager;
import com.kylinolap.metadata.model.SegmentStatusEnum;
import com.kylinolap.metadata.model.TblColRef;
+import com.kylinolap.metadata.realization.IRealization;
+import com.kylinolap.metadata.realization.IRealizationProvider;
+import com.kylinolap.metadata.realization.RealizationType;
/**
* @author honma
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a1afffdd/job/src/main/java/com/kylinolap/job/CubeMetadataUpgrade.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/com/kylinolap/job/CubeMetadataUpgrade.java b/job/src/main/java/com/kylinolap/job/CubeMetadataUpgrade.java
index 1ea0317..b9775a7 100644
--- a/job/src/main/java/com/kylinolap/job/CubeMetadataUpgrade.java
+++ b/job/src/main/java/com/kylinolap/job/CubeMetadataUpgrade.java
@@ -11,6 +11,7 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -93,7 +94,7 @@ public class CubeMetadataUpgrade {
upgradeCubeDesc();
upgradeProjectInstance();
upgradeCubeInstance();
- upgradeJobInstance();
+// upgradeJobInstance();
verify();
@@ -103,6 +104,8 @@ public class CubeMetadataUpgrade {
MetadataManager.getInstance(config).reload();
CubeDescManager.clearCache();
CubeDescManager.getInstance(config);
+ CubeManager.getInstance(config);
+ ProjectManager.getInstance(config);
}
private List<String> listResourceStore(String pathRoot) {
@@ -235,6 +238,11 @@ public class CubeMetadataUpgrade {
if (count > 1) {
errorMsgs.add("There are more than 1 table named with '" + table + "' in different database; The program couldn't determine, randomly pick '" + result + "'");
}
+
+ if (count ==0) {
+ errorMsgs.add("There is no table named with '" + table + "'");
+ }
+
return result;
}
@@ -250,7 +258,6 @@ public class CubeMetadataUpgrade {
newPrj.setOwner(oldPrj.getOwner());
newPrj.setDescription(oldPrj.getDescription());
newPrj.setLastModified(oldPrj.getLastModified());
- newPrj.setLastUpdateTime(oldPrj.getLastUpdateTime());
newPrj.setCreateTimeUTC(RootPersistentEntity.parseTime(oldPrj.getCreateTime()));
newPrj.setStatus(oldPrj.getStatus());
List<RealizationEntry> realizationEntries = Lists.newArrayList();
@@ -261,7 +268,6 @@ public class CubeMetadataUpgrade {
realizationEntries.add(entry);
}
newPrj.setRealizationEntries(realizationEntries);
- newPrj.getCreateTimeUTC();
Set<String> tables = Sets.newHashSet();
for (String table : oldPrj.getTables()) {
@@ -297,6 +303,7 @@ public class CubeMetadataUpgrade {
newInstance.setUuid(cubeInstance.getUuid());
newInstance.setVersion(cubeInstance.getVersion());
newInstance.setCreateTimeUTC(RootPersistentEntity.parseTime(cubeInstance.getCreateTime()));
+ newInstance.setLastModified(cubeInstance.getLastModified());
//status
if(cubeInstance.getStatus() == CubeStatusEnum.BUILDING) {
@@ -336,8 +343,23 @@ public class CubeMetadataUpgrade {
newSeg.setLastBuildJobID(segment.getLastBuildJobID());
newSeg.setCreateTimeUTC(RootPersistentEntity.parseTime(segment.getCreateTime()));
newSeg.setBinarySignature(segment.getBinarySignature());
- newSeg.setDictionaries((ConcurrentHashMap<String, String>)segment.getDictionaries());
- newSeg.setSnapshots((ConcurrentHashMap<String, String>)segment.getSnapshots());
+
+ ConcurrentHashMap<String, String> newDictionaries = new ConcurrentHashMap<String, String>();
+
+ for (Map.Entry<String, String> e : segment.getDictionaries().entrySet()) {
+ String key = e.getKey();
+ String[] tableCol = StringUtils.split(key, "/");
+ key = appendDBName(tableCol[0]) + "/" + tableCol[1];
+ newDictionaries.put(key, e.getValue());
+ }
+ newSeg.setDictionaries(newDictionaries);
+
+ ConcurrentHashMap<String, String> newSnapshots = new ConcurrentHashMap<String, String>();
+
+ for(Map.Entry<String, String> e: segment.getSnapshots().entrySet()) {
+ newSnapshots.put(appendDBName(e.getKey()), e.getValue());
+ }
+ newSeg.setSnapshots(newSnapshots);
}
newInstance.setSegments(newSegments);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a1afffdd/job/src/main/java/com/kylinolap/job/JoinedFlatTable.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/com/kylinolap/job/JoinedFlatTable.java b/job/src/main/java/com/kylinolap/job/JoinedFlatTable.java
index 5a3604a..d5f684c 100644
--- a/job/src/main/java/com/kylinolap/job/JoinedFlatTable.java
+++ b/job/src/main/java/com/kylinolap/job/JoinedFlatTable.java
@@ -185,8 +185,8 @@ public class JoinedFlatTable {
CubeDesc cubeDesc = desc.getCubeDesc();
- if (cubeDesc.getFilterCondition() != null && cubeDesc.getFilterCondition().equals("") == false) {
- whereBuilder.append(" (").append(cubeDesc.getFilterCondition()).append(") ");
+ if (cubeDesc.getModel().getFilterCondition() != null && cubeDesc.getModel().getFilterCondition().equals("") == false) {
+ whereBuilder.append(" (").append(cubeDesc.getModel().getFilterCondition()).append(") ");
hasCondition = true;
}
@@ -197,7 +197,7 @@ public class JoinedFlatTable {
long dateEnd = cubeSegment.getDateRangeEnd();
if (!(dateStart == 0 && dateEnd == Long.MAX_VALUE)) {
- String partitionColumnName = cubeDesc.getCubePartitionDesc().getPartitionDateColumn();
+ String partitionColumnName = cubeDesc.getModel().getPartitionDesc().getPartitionDateColumn();
int indexOfDot = partitionColumnName.lastIndexOf(".");
// convert to use table alias;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a1afffdd/job/src/main/java/com/kylinolap/job/cube/CubingJobBuilder.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/com/kylinolap/job/cube/CubingJobBuilder.java b/job/src/main/java/com/kylinolap/job/cube/CubingJobBuilder.java
index fe4235d..75e4d54 100644
--- a/job/src/main/java/com/kylinolap/job/cube/CubingJobBuilder.java
+++ b/job/src/main/java/com/kylinolap/job/cube/CubingJobBuilder.java
@@ -211,7 +211,7 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
private void appendMapReduceParameters(StringBuilder builder, CubeSegment seg) {
try {
- String jobConf = engineConfig.getHadoopJobConfFilePath(seg.getCubeDesc().getCapacity());
+ String jobConf = engineConfig.getHadoopJobConfFilePath(seg.getCubeDesc().getModel().getCapacity());
if (jobConf != null && jobConf.length() > 0) {
builder.append(" -conf ").append(jobConf);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a1afffdd/job/src/main/java/com/kylinolap/job/engine/JobEngineConfig.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/com/kylinolap/job/engine/JobEngineConfig.java b/job/src/main/java/com/kylinolap/job/engine/JobEngineConfig.java
index bf502a3..1e49cea 100644
--- a/job/src/main/java/com/kylinolap/job/engine/JobEngineConfig.java
+++ b/job/src/main/java/com/kylinolap/job/engine/JobEngineConfig.java
@@ -27,8 +27,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.kylinolap.common.KylinConfig;
-import com.kylinolap.cube.model.CubeDesc.RealizationCapacity;
import com.kylinolap.job.tools.OptionsHelper;
+import com.kylinolap.metadata.model.DataModelDesc.RealizationCapacity;
/**
* @author ysong1
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a1afffdd/job/src/main/java/com/kylinolap/job/hadoop/cube/RangeKeyDistributionJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/com/kylinolap/job/hadoop/cube/RangeKeyDistributionJob.java b/job/src/main/java/com/kylinolap/job/hadoop/cube/RangeKeyDistributionJob.java
index 8364309..d48d138 100644
--- a/job/src/main/java/com/kylinolap/job/hadoop/cube/RangeKeyDistributionJob.java
+++ b/job/src/main/java/com/kylinolap/job/hadoop/cube/RangeKeyDistributionJob.java
@@ -33,9 +33,9 @@ import org.slf4j.LoggerFactory;
import com.kylinolap.common.KylinConfig;
import com.kylinolap.cube.CubeInstance;
import com.kylinolap.cube.CubeManager;
-import com.kylinolap.cube.model.CubeDesc.RealizationCapacity;
import com.kylinolap.job.constant.BatchConstants;
import com.kylinolap.job.hadoop.AbstractHadoopJob;
+import com.kylinolap.metadata.model.DataModelDesc.RealizationCapacity;
/**
* @author xjiang, ysong1
@@ -97,7 +97,7 @@ public class RangeKeyDistributionJob extends AbstractHadoopJob {
String cubeName = getOptionValue(OPTION_CUBE_NAME).toUpperCase();
CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
CubeInstance cube = cubeMgr.getCube(cubeName);
- RealizationCapacity realizationCapacity = cube.getDescriptor().getCapacity();
+ RealizationCapacity realizationCapacity = cube.getDescriptor().getModel().getCapacity();
job.getConfiguration().set(BatchConstants.CUBE_CAPACITY, realizationCapacity.toString());
return waitForCompletion(job);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a1afffdd/job/src/main/java/com/kylinolap/job/hadoop/hive/CubeJoinedFlatTableDesc.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/com/kylinolap/job/hadoop/hive/CubeJoinedFlatTableDesc.java b/job/src/main/java/com/kylinolap/job/hadoop/hive/CubeJoinedFlatTableDesc.java
index 675ebe9..230aa7f 100644
--- a/job/src/main/java/com/kylinolap/job/hadoop/hive/CubeJoinedFlatTableDesc.java
+++ b/job/src/main/java/com/kylinolap/job/hadoop/hive/CubeJoinedFlatTableDesc.java
@@ -184,8 +184,8 @@ public class CubeJoinedFlatTableDesc implements IJoinedFlatTableDesc {
}
@Override
- public CubeDesc.RealizationCapacity getCapacity() {
- return cubeDesc.getCapacity();
+ public DataModelDesc.RealizationCapacity getCapacity() {
+ return cubeDesc.getModel().getCapacity();
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a1afffdd/job/src/main/java/com/kylinolap/job/hadoop/hive/IIJoinedFlatTableDesc.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/com/kylinolap/job/hadoop/hive/IIJoinedFlatTableDesc.java b/job/src/main/java/com/kylinolap/job/hadoop/hive/IIJoinedFlatTableDesc.java
index 635dacb..121e0cb 100644
--- a/job/src/main/java/com/kylinolap/job/hadoop/hive/IIJoinedFlatTableDesc.java
+++ b/job/src/main/java/com/kylinolap/job/hadoop/hive/IIJoinedFlatTableDesc.java
@@ -5,7 +5,6 @@ import java.util.Map;
import java.util.TreeMap;
import com.google.common.collect.Lists;
-import com.kylinolap.cube.model.CubeDesc;
import com.kylinolap.invertedindex.model.IIDesc;
import com.kylinolap.metadata.model.DataModelDesc;
import com.kylinolap.metadata.model.JoinDesc;
@@ -68,8 +67,8 @@ public class IIJoinedFlatTableDesc implements IJoinedFlatTableDesc {
}
@Override
- public CubeDesc.RealizationCapacity getCapacity() {
- return CubeDesc.RealizationCapacity.SMALL;
+ public DataModelDesc.RealizationCapacity getCapacity() {
+ return DataModelDesc.RealizationCapacity.SMALL;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a1afffdd/job/src/main/java/com/kylinolap/job/hadoop/hive/IJoinedFlatTableDesc.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/com/kylinolap/job/hadoop/hive/IJoinedFlatTableDesc.java b/job/src/main/java/com/kylinolap/job/hadoop/hive/IJoinedFlatTableDesc.java
index b9779cc..5c9d59e 100644
--- a/job/src/main/java/com/kylinolap/job/hadoop/hive/IJoinedFlatTableDesc.java
+++ b/job/src/main/java/com/kylinolap/job/hadoop/hive/IJoinedFlatTableDesc.java
@@ -1,10 +1,9 @@
package com.kylinolap.job.hadoop.hive;
-import com.kylinolap.cube.model.CubeDesc;
-import com.kylinolap.metadata.model.DataModelDesc;
-
import java.util.List;
+import com.kylinolap.metadata.model.DataModelDesc;
+
/**
* Created by Hongbin Ma(Binmahone) on 12/30/14.
*/
@@ -20,7 +19,7 @@ public interface IJoinedFlatTableDesc {
public DataModelDesc getDataModel();
- public CubeDesc.RealizationCapacity getCapacity();
+ public DataModelDesc.RealizationCapacity getCapacity();
public String getTableAlias(String tableName);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a1afffdd/job/src/main/java/com/kylinolap/job/invertedindex/IIJobBuilder.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/com/kylinolap/job/invertedindex/IIJobBuilder.java b/job/src/main/java/com/kylinolap/job/invertedindex/IIJobBuilder.java
index 5e28b8f..b370ec6 100644
--- a/job/src/main/java/com/kylinolap/job/invertedindex/IIJobBuilder.java
+++ b/job/src/main/java/com/kylinolap/job/invertedindex/IIJobBuilder.java
@@ -21,13 +21,13 @@ import java.util.Date;
import java.util.TimeZone;
import com.google.common.base.Preconditions;
-import com.kylinolap.cube.model.CubeDesc.RealizationCapacity;
import com.kylinolap.invertedindex.IISegment;
import com.kylinolap.job.AbstractJobBuilder;
import com.kylinolap.job.common.HadoopShellExecutable;
import com.kylinolap.job.common.MapReduceExecutable;
import com.kylinolap.job.constant.ExecutableConstants;
import com.kylinolap.job.engine.JobEngineConfig;
+import com.kylinolap.job.execution.AbstractExecutable;
import com.kylinolap.job.hadoop.dict.CreateInvertedIndexDictionaryJob;
import com.kylinolap.job.hadoop.hive.IIJoinedFlatTableDesc;
import com.kylinolap.job.hadoop.invertedindex.IIBulkLoadJob;
@@ -35,7 +35,7 @@ import com.kylinolap.job.hadoop.invertedindex.IICreateHFileJob;
import com.kylinolap.job.hadoop.invertedindex.IICreateHTableJob;
import com.kylinolap.job.hadoop.invertedindex.IIDistinctColumnsJob;
import com.kylinolap.job.hadoop.invertedindex.InvertedIndexJob;
-import com.kylinolap.job.execution.AbstractExecutable;
+import com.kylinolap.metadata.model.DataModelDesc.RealizationCapacity;
/**
* Created by shaoshi on 1/15/15.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a1afffdd/job/src/test/java/com/kylinolap/job/BuildCubeWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/BuildCubeWithEngineTest.java b/job/src/test/java/com/kylinolap/job/BuildCubeWithEngineTest.java
index 89bf584..8e62685 100644
--- a/job/src/test/java/com/kylinolap/job/BuildCubeWithEngineTest.java
+++ b/job/src/test/java/com/kylinolap/job/BuildCubeWithEngineTest.java
@@ -211,7 +211,7 @@ public class BuildCubeWithEngineTest {
List<String> result = Lists.newArrayList();
final String cubeName = "test_kylin_cube_without_slr_left_join_empty";
// this cube's start date is 0, end date is 20120601000000
- long dateStart = cubeManager.getCube(cubeName).getDescriptor().getCubePartitionDesc().getPartitionDateStart();
+ long dateStart = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
long dateEnd = f.parse("2012-06-01").getTime();
clearSegment(cubeName);
@@ -232,7 +232,7 @@ public class BuildCubeWithEngineTest {
SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
f.setTimeZone(TimeZone.getTimeZone("GMT"));
- long dateStart = cubeManager.getCube(cubeName).getDescriptor().getCubePartitionDesc().getPartitionDateStart();
+ long dateStart = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
long dateEnd = f.parse("2050-11-12").getTime();
// this cube's start date is 0, end date is 20501112000000
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a1afffdd/job/src/test/java/com/kylinolap/job/tools/ColumnCardinalityMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/tools/ColumnCardinalityMapperTest.java b/job/src/test/java/com/kylinolap/job/tools/ColumnCardinalityMapperTest.java
index 34b6f0c..da721fc 100644
--- a/job/src/test/java/com/kylinolap/job/tools/ColumnCardinalityMapperTest.java
+++ b/job/src/test/java/com/kylinolap/job/tools/ColumnCardinalityMapperTest.java
@@ -37,7 +37,6 @@ import org.junit.Test;
import com.kylinolap.common.hll.HyperLogLogPlusCounter;
import com.kylinolap.job.hadoop.cardinality.ColumnCardinalityMapper;
-import com.kylinolap.job.hadoop.cardinality.HiveColumnCardinalityJob;
/**
* @author ysong1
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a1afffdd/metadata/src/main/java/com/kylinolap/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/model/DataModelDesc.java b/metadata/src/main/java/com/kylinolap/metadata/model/DataModelDesc.java
index 17c1f36..d2af2e7 100644
--- a/metadata/src/main/java/com/kylinolap/metadata/model/DataModelDesc.java
+++ b/metadata/src/main/java/com/kylinolap/metadata/model/DataModelDesc.java
@@ -1,13 +1,16 @@
package com.kylinolap.metadata.model;
-import java.util.*;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Map;
-import com.google.common.collect.Sets;
import org.apache.commons.lang.ArrayUtils;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.collect.Sets;
import com.kylinolap.common.persistence.ResourceStore;
import com.kylinolap.common.persistence.RootPersistentEntity;
import com.kylinolap.common.util.StringUtil;
@@ -16,6 +19,10 @@ import com.kylinolap.metadata.MetadataConstances;
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class DataModelDesc extends RootPersistentEntity {
+ public static enum RealizationCapacity {
+ SMALL, MEDIUM, LARGE
+ }
+
@JsonProperty("name")
private String name;
@@ -25,6 +32,14 @@ public class DataModelDesc extends RootPersistentEntity {
@JsonProperty("lookups")
private LookupDesc[] lookups;
+ @JsonProperty("filter_condition")
+ private String filterCondition;
+ @JsonProperty("partition_desc")
+ PartitionDesc partitionDesc;
+
+ @JsonProperty("capacity")
+ private RealizationCapacity capacity = RealizationCapacity.MEDIUM;
+
public String getName() {
return name;
}
@@ -60,6 +75,31 @@ public class DataModelDesc extends RootPersistentEntity {
public boolean isFactTable(String factTable) {
return this.factTable.equalsIgnoreCase(factTable);
}
+
+
+ public String getFilterCondition() {
+ return filterCondition;
+ }
+
+ public void setFilterCondition(String filterCondition) {
+ this.filterCondition = filterCondition;
+ }
+
+ public PartitionDesc getPartitionDesc() {
+ return partitionDesc;
+ }
+
+ public void setPartitionDesc(PartitionDesc partitionDesc) {
+ this.partitionDesc = partitionDesc;
+ }
+
+ public RealizationCapacity getCapacity() {
+ return capacity;
+ }
+
+ public void setCapacity(RealizationCapacity capacity) {
+ this.capacity = capacity;
+ }
public TblColRef findPKByFK(TblColRef fk) {
assert isFactTable(fk.getTable());
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a1afffdd/metadata/src/main/java/com/kylinolap/metadata/model/PartitionDesc.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/model/PartitionDesc.java b/metadata/src/main/java/com/kylinolap/metadata/model/PartitionDesc.java
new file mode 100644
index 0000000..b0d8216
--- /dev/null
+++ b/metadata/src/main/java/com/kylinolap/metadata/model/PartitionDesc.java
@@ -0,0 +1,100 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.metadata.model;
+
+import java.util.Map;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.kylinolap.common.util.StringSplitter;
+import com.kylinolap.metadata.model.TblColRef;
+
+/**
+ * @author xduo
+ *
+ */
+@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
+public class PartitionDesc {
+
+ public static enum PartitionType {
+ APPEND, //
+ UPDATE_INSERT // not used since 0.7.1
+ }
+
+ @JsonProperty("partition_date_column")
+ private String partitionDateColumn;
+ @JsonProperty("partition_date_start")
+ private long partitionDateStart = 0L;
+ @JsonProperty("partition_type")
+ private PartitionType partitionType = PartitionType.APPEND;
+
+ private TblColRef partitionDateColumnRef;
+
+ public void init(Map<String, Map<String, TblColRef>> columnMap) {
+ if (null != partitionDateColumn) {
+ partitionDateColumn = partitionDateColumn.toUpperCase();
+
+ String[] columns = StringSplitter.split(partitionDateColumn, ".");
+
+ if (null != columns && columns.length == 3) {
+ String tableName = columns[0].toUpperCase() + "." + columns[1].toUpperCase();
+ Map<String, TblColRef> cols = columnMap.get(tableName);
+ if (cols != null) {
+ partitionDateColumnRef = cols.get(columns[2].toUpperCase());
+ } else {
+ throw new IllegalStateException("The table '" + tableName + "' provided in 'partition_date_column' doesn't exist.");
+ }
+ } else {
+ throw new IllegalStateException("The 'partition_date_column' format is invalid: " + partitionDateColumn + ", it should be {db}.{table}.{column}.");
+ }
+ }
+ }
+
+ public boolean isPartitioned() {
+ return partitionDateColumnRef != null;
+ }
+
+ public String getPartitionDateColumn() {
+ return partitionDateColumn;
+ }
+
+ public void setPartitionDateColumn(String partitionDateColumn) {
+ this.partitionDateColumn = partitionDateColumn;
+ }
+
+ public long getPartitionDateStart() {
+ return partitionDateStart;
+ }
+
+ public void setPartitionDateStart(long partitionDateStart) {
+ this.partitionDateStart = partitionDateStart;
+ }
+
+ public PartitionType getCubePartitionType() {
+ return partitionType;
+ }
+
+ public void setCubePartitionType(PartitionType partitionType) {
+ this.partitionType = partitionType;
+ }
+
+ public TblColRef getPartitionDateColumnRef() {
+ return partitionDateColumnRef;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a1afffdd/metadata/src/main/java/com/kylinolap/metadata/project/ProjectInstance.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/project/ProjectInstance.java b/metadata/src/main/java/com/kylinolap/metadata/project/ProjectInstance.java
index fa0015a..01d1770 100644
--- a/metadata/src/main/java/com/kylinolap/metadata/project/ProjectInstance.java
+++ b/metadata/src/main/java/com/kylinolap/metadata/project/ProjectInstance.java
@@ -216,17 +216,6 @@ public class ProjectInstance extends RootPersistentEntity {
this.owner = owner;
}
- /**
- * @deprecated use lastModified instead
- */
- public String getLastUpdateTime() {
- return lastUpdateTime;
- }
-
- public void setLastUpdateTime(String lastUpdateTime) {
- this.lastUpdateTime = lastUpdateTime;
- }
-
public void recordUpdateTime(long timeMillis) {
this.lastUpdateTime = formatTime(timeMillis);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a1afffdd/server/src/main/java/com/kylinolap/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/com/kylinolap/rest/service/CubeService.java b/server/src/main/java/com/kylinolap/rest/service/CubeService.java
index 45ccb54..30ca7c0 100644
--- a/server/src/main/java/com/kylinolap/rest/service/CubeService.java
+++ b/server/src/main/java/com/kylinolap/rest/service/CubeService.java
@@ -17,9 +17,6 @@ package com.kylinolap.rest.service;
import java.io.IOException;
import java.net.UnknownHostException;
-import java.text.DateFormat;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
@@ -52,10 +49,10 @@ import com.kylinolap.cube.model.CubeDesc;
import com.kylinolap.job.common.HadoopShellExecutable;
import com.kylinolap.job.cube.CubingJob;
import com.kylinolap.job.exception.JobException;
+import com.kylinolap.job.execution.DefaultChainedExecutable;
import com.kylinolap.job.execution.ExecutableState;
import com.kylinolap.job.hadoop.cardinality.HiveColumnCardinalityJob;
import com.kylinolap.job.hadoop.cardinality.HiveColumnCardinalityUpdateJob;
-import com.kylinolap.job.execution.DefaultChainedExecutable;
import com.kylinolap.metadata.MetadataConstances;
import com.kylinolap.metadata.MetadataManager;
import com.kylinolap.metadata.model.SegmentStatusEnum;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a1afffdd/storage/src/main/java/com/kylinolap/storage/hbase/CubeStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/com/kylinolap/storage/hbase/CubeStorageEngine.java b/storage/src/main/java/com/kylinolap/storage/hbase/CubeStorageEngine.java
index c8190ab..c9ada68 100644
--- a/storage/src/main/java/com/kylinolap/storage/hbase/CubeStorageEngine.java
+++ b/storage/src/main/java/com/kylinolap/storage/hbase/CubeStorageEngine.java
@@ -568,7 +568,7 @@ public class CubeStorageEngine implements IStorageEngine {
}
private void dropUnhitSegments(List<HBaseKeyRange> scans) {
- if (cubeDesc.getCubePartitionDesc().isPartitioned()) {
+ if (cubeDesc.getModel().getPartitionDesc().isPartitioned()) {
Iterator<HBaseKeyRange> iterator = scans.iterator();
while (iterator.hasNext()) {
HBaseKeyRange scan = iterator.next();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a1afffdd/storage/src/main/java/com/kylinolap/storage/hbase/HBaseKeyRange.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/com/kylinolap/storage/hbase/HBaseKeyRange.java b/storage/src/main/java/com/kylinolap/storage/hbase/HBaseKeyRange.java
index b163297..2d27986 100644
--- a/storage/src/main/java/com/kylinolap/storage/hbase/HBaseKeyRange.java
+++ b/storage/src/main/java/com/kylinolap/storage/hbase/HBaseKeyRange.java
@@ -114,7 +114,7 @@ public class HBaseKeyRange implements Comparable<HBaseKeyRange> {
stopValues.put(column, dimRange.getEndValue());
fuzzyValues.put(column, dimRange.getEqualValues());
- TblColRef partitionDateColumnRef = cubeSeg.getCubeDesc().getCubePartitionDesc().getPartitionDateColumnRef();
+ TblColRef partitionDateColumnRef = cubeSeg.getCubeDesc().getModel().getPartitionDesc().getPartitionDateColumnRef();
if (column.equals(partitionDateColumnRef)) {
initPartitionRange(dimRange);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a1afffdd/storage/src/test/java/com/kylinolap/storage/MiniClusterTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/com/kylinolap/storage/MiniClusterTest.java b/storage/src/test/java/com/kylinolap/storage/MiniClusterTest.java
index 8494e97..ddb959f 100644
--- a/storage/src/test/java/com/kylinolap/storage/MiniClusterTest.java
+++ b/storage/src/test/java/com/kylinolap/storage/MiniClusterTest.java
@@ -1,18 +1,11 @@
package com.kylinolap.storage;
-import java.io.BufferedReader;
import java.io.File;
-import java.io.InputStreamReader;
-import java.net.MalformedURLException;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.MiniHBaseCluster;
-import org.apache.hadoop.hbase.client.HTablePool;
-import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a1afffdd/storage/src/test/java/com/kylinolap/storage/minicluster/HiveJDBCClientTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/com/kylinolap/storage/minicluster/HiveJDBCClientTest.java b/storage/src/test/java/com/kylinolap/storage/minicluster/HiveJDBCClientTest.java
index bbe76bc..cfb2667 100644
--- a/storage/src/test/java/com/kylinolap/storage/minicluster/HiveJDBCClientTest.java
+++ b/storage/src/test/java/com/kylinolap/storage/minicluster/HiveJDBCClientTest.java
@@ -9,10 +9,6 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
/**
* A simple test case to check whether the Hive JDBC interface can fulfill Kylin's need;
* Before run it, you need startup the hive server on localhost: $HIVE_HOME/bin/hiveserver2
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a1afffdd/storage/src/test/java/com/kylinolap/storage/minicluster/HiveMiniClusterTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/com/kylinolap/storage/minicluster/HiveMiniClusterTest.java b/storage/src/test/java/com/kylinolap/storage/minicluster/HiveMiniClusterTest.java
index 0dc4608..2af2696 100644
--- a/storage/src/test/java/com/kylinolap/storage/minicluster/HiveMiniClusterTest.java
+++ b/storage/src/test/java/com/kylinolap/storage/minicluster/HiveMiniClusterTest.java
@@ -4,9 +4,7 @@ import java.io.File;
import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
-import java.sql.ResultSet;
import java.sql.SQLException;
-import java.sql.Statement;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.hdfs.MiniDFSCluster;
@@ -14,10 +12,6 @@ import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.service.HiveInterface;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MiniMRCluster;
-import org.apache.thrift.TException;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
/**
* This is a test case to verify whether the query can be executed on Hive minicluster;