You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/01/31 13:01:31 UTC
[04/50] incubator-kylin git commit: KYLIN-540 Data model upgrade for
legacy cube descs;
rename CubeSegment and CubeInstance sourceRecords, sourceRecordsSize ==>
inputRecords, inputRecordsSize
KYLIN-540 Data model upgrade for legacy cube descs; rename CubeSegment and CubeInstance sourceRecords, sourceRecordsSize ==> inputRecords, inputRecordsSize
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/be56ebff
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/be56ebff
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/be56ebff
Branch: refs/heads/inverted-index
Commit: be56ebffbc19aa51a7efe06e1ed534e1c47b194b
Parents: 89753f0
Author: Shao Feng, Shi <sh...@hotmail.com>
Authored: Wed Jan 28 09:40:48 2015 +0800
Committer: Shao Feng, Shi <sh...@hotmail.com>
Committed: Wed Jan 28 09:40:48 2015 +0800
----------------------------------------------------------------------
.../java/com/kylinolap/cube/CubeInstance.java | 31 +-
.../java/com/kylinolap/cube/CubeManager.java | 7 +-
.../java/com/kylinolap/cube/CubeSegment.java | 54 +--
.../cube/model/v1/CubeBuildTypeEnum.java | 32 ++
.../kylinolap/cube/model/v1/CubeInstance.java | 425 +++++++++++++++++++
.../kylinolap/cube/model/v1/CubeSegment.java | 306 +++++++++++++
.../cube/model/v1/CubeSegmentStatusEnum.java | 25 ++
.../cube/model/v1/CubeSegmentTypeEnum.java | 25 ++
.../kylinolap/cube/model/v1/CubeStatusEnum.java | 22 +
.../cube/model/v1/ProjectInstance.java | 4 -
.../kylinolap/cube/SegmentManagementTest.java | 4 +-
.../com/kylinolap/invertedindex/IIInstance.java | 12 +-
.../com/kylinolap/invertedindex/IISegment.java | 24 +-
.../com/kylinolap/job/CubeMetadataUpgrade.java | 116 ++++-
.../job/cube/UpdateCubeInfoAfterBuildStep.java | 4 +-
.../job/cube/UpdateCubeInfoAfterMergeStep.java | 8 +-
.../metadata/project/ProjectInstance.java | 19 -
.../metadata/project/ProjectManager.java | 2 +-
.../com/kylinolap/rest/service/CubeService.java | 7 +-
19 files changed, 994 insertions(+), 133 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/be56ebff/cube/src/main/java/com/kylinolap/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/CubeInstance.java b/cube/src/main/java/com/kylinolap/cube/CubeInstance.java
index bcd9bad..919daf4 100644
--- a/cube/src/main/java/com/kylinolap/cube/CubeInstance.java
+++ b/cube/src/main/java/com/kylinolap/cube/CubeInstance.java
@@ -77,9 +77,6 @@ public class CubeInstance extends RootPersistentEntity implements IRealization {
@JsonProperty("segments")
private List<CubeSegment> segments = new ArrayList<CubeSegment>();
- @JsonProperty("create_time")
- private String createTime;
-
@JsonProperty("create_time_utc")
private long createTimeUTC;
@@ -208,23 +205,23 @@ public class CubeInstance extends RootPersistentEntity implements IRealization {
return sizeKb;
}
- @JsonProperty("source_records_count")
- public long getSourceRecordCount() {
+ @JsonProperty("input_records_count")
+ public long getInputRecordCount() {
long sizeRecordCount = 0L;
for (CubeSegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) {
- sizeRecordCount += cubeSegment.getSourceRecords();
+ sizeRecordCount += cubeSegment.getInputRecords();
}
return sizeRecordCount;
}
- @JsonProperty("source_records_size")
- public long getSourceRecordSize() {
+ @JsonProperty("input_records_size")
+ public long getInputRecordSize() {
long sizeRecordSize = 0L;
for (CubeSegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) {
- sizeRecordSize += cubeSegment.getSourceRecordsSize();
+ sizeRecordSize += cubeSegment.getInputRecordsSize();
}
return sizeRecordSize;
@@ -376,23 +373,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization {
return null;
}
- /**
- * @deprecated use createTimeUTC instead
- * @return
- */
- public String getCreateTime() {
- return createTime;
- }
-
- public void setCreateTime(String createTime) {
- this.createTime = createTime;
- }
-
public long getCreateTimeUTC() {
- if(createTimeUTC ==0 && createTime !=null) {
- createTimeUTC = parseTime(createTime);
- }
-
return createTimeUTC;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/be56ebff/cube/src/main/java/com/kylinolap/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/CubeManager.java b/cube/src/main/java/com/kylinolap/cube/CubeManager.java
index ebf35c3..be4ac45 100644
--- a/cube/src/main/java/com/kylinolap/cube/CubeManager.java
+++ b/cube/src/main/java/com/kylinolap/cube/CubeManager.java
@@ -72,7 +72,7 @@ public class CubeManager implements IRealizationProvider {
private static String ALPHA_NUM = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";
private static int HBASE_TABLE_LENGTH = 10;
- private static final Serializer<CubeInstance> CUBE_SERIALIZER = new JsonSerializer<CubeInstance>(CubeInstance.class);
+ public static final Serializer<CubeInstance> CUBE_SERIALIZER = new JsonSerializer<CubeInstance>(CubeInstance.class);
private static final Logger logger = LoggerFactory.getLogger(CubeManager.class);
@@ -397,8 +397,8 @@ public class CubeManager implements IRealizationProvider {
cubeSegment.setLastBuildJobID(jobUuid);
cubeSegment.setLastBuildTime(lastBuildTime);
cubeSegment.setSizeKB(sizeKB);
- cubeSegment.setSourceRecords(sourceRecordCount);
- cubeSegment.setSourceRecordsSize(sourceRecordsSize);
+ cubeSegment.setInputRecords(sourceRecordCount);
+ cubeSegment.setInputRecordsSize(sourceRecordsSize);
cubeSegment.setStatus(SegmentStatusEnum.READY);
cubeInstance.setStatus(RealizationStatusEnum.READY);
this.updateCube(cubeInstance);
@@ -483,7 +483,6 @@ public class CubeManager implements IRealizationProvider {
segment.setUuid(UUID.randomUUID().toString());
segment.setName(incrementalSegName);
Date creatTime = new Date();
- segment.setCreateTime(DateStrDictionary.dateToString(creatTime));
segment.setCreateTimeUTC(creatTime.getTime());
segment.setDateRangeStart(startDate);
segment.setDateRangeEnd(endDate);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/be56ebff/cube/src/main/java/com/kylinolap/cube/CubeSegment.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/CubeSegment.java b/cube/src/main/java/com/kylinolap/cube/CubeSegment.java
index ff1001e..7f46a07 100644
--- a/cube/src/main/java/com/kylinolap/cube/CubeSegment.java
+++ b/cube/src/main/java/com/kylinolap/cube/CubeSegment.java
@@ -26,10 +26,9 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonBackReference;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Objects;
-import com.kylinolap.common.persistence.RootPersistentEntity;
import com.kylinolap.cube.model.CubeDesc;
-import com.kylinolap.dict.ISegment;
import com.kylinolap.dict.Dictionary;
+import com.kylinolap.dict.ISegment;
import com.kylinolap.metadata.model.SegmentStatusEnum;
import com.kylinolap.metadata.model.TblColRef;
@@ -52,17 +51,14 @@ public class CubeSegment implements Comparable<CubeSegment>, ISegment {
private SegmentStatusEnum status;
@JsonProperty("size_kb")
private long sizeKB;
- @JsonProperty("source_records")
- private long sourceRecords;
- @JsonProperty("source_records_size")
- private long sourceRecordsSize;
+ @JsonProperty("input_records")
+ private long inputRecords;
+ @JsonProperty("input_records_size")
+ private long inputRecordsSize;
@JsonProperty("last_build_time")
private long lastBuildTime;
@JsonProperty("last_build_job_id")
private String lastBuildJobID;
- @JsonProperty("create_time")
- private String createTime;
-
@JsonProperty("create_time_utc")
private long createTimeUTC;
@@ -146,20 +142,20 @@ public class CubeSegment implements Comparable<CubeSegment>, ISegment {
this.sizeKB = sizeKB;
}
- public long getSourceRecords() {
- return sourceRecords;
+ public long getInputRecords() {
+ return inputRecords;
}
- public void setSourceRecords(long sourceRecords) {
- this.sourceRecords = sourceRecords;
+ public void setInputRecords(long inputRecords) {
+ this.inputRecords = inputRecords;
}
- public long getSourceRecordsSize() {
- return sourceRecordsSize;
+ public long getInputRecordsSize() {
+ return inputRecordsSize;
}
- public void setSourceRecordsSize(long sourceRecordsSize) {
- this.sourceRecordsSize = sourceRecordsSize;
+ public void setInputRecordsSize(long inputRecordsSize) {
+ this.inputRecordsSize = inputRecordsSize;
}
public long getLastBuildTime() {
@@ -178,22 +174,8 @@ public class CubeSegment implements Comparable<CubeSegment>, ISegment {
this.lastBuildJobID = lastBuildJobID;
}
- /**
- * @deprecated
- * @return
- */
- public String getCreateTime() {
- return createTime;
- }
-
- public void setCreateTime(String createTime) {
- this.createTime = createTime;
- }
public long getCreateTimeUTC() {
- if (createTimeUTC == 0 && createTime != null) {
- createTimeUTC = RootPersistentEntity.parseTime(createTime);
- }
return createTimeUTC;
}
@@ -340,4 +322,14 @@ public class CubeSegment implements Comparable<CubeSegment>, ISegment {
public Dictionary<?> getDictionary(TblColRef col) {
return CubeManager.getInstance(this.getCubeInstance().getConfig()).getDictionary(this, col);
}
+
+ public void setDictionaries(ConcurrentHashMap<String, String> dictionaries) {
+ this.dictionaries = dictionaries;
+ }
+
+ public void setSnapshots(ConcurrentHashMap<String, String> snapshots) {
+ this.snapshots = snapshots;
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/be56ebff/cube/src/main/java/com/kylinolap/cube/model/v1/CubeBuildTypeEnum.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/model/v1/CubeBuildTypeEnum.java b/cube/src/main/java/com/kylinolap/cube/model/v1/CubeBuildTypeEnum.java
new file mode 100644
index 0000000..59c40b1
--- /dev/null
+++ b/cube/src/main/java/com/kylinolap/cube/model/v1/CubeBuildTypeEnum.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.cube.model.v1;
+
+/**
+ * @author xduo
+ *
+ */
+public enum CubeBuildTypeEnum {
+ /**
+ * rebuild a segment or incremental build
+ */
+ BUILD,
+ /**
+ * merge segments
+ */
+ MERGE;
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/be56ebff/cube/src/main/java/com/kylinolap/cube/model/v1/CubeInstance.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/model/v1/CubeInstance.java b/cube/src/main/java/com/kylinolap/cube/model/v1/CubeInstance.java
new file mode 100644
index 0000000..6d50c1b
--- /dev/null
+++ b/cube/src/main/java/com/kylinolap/cube/model/v1/CubeInstance.java
@@ -0,0 +1,425 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.cube.model.v1;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonManagedReference;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Objects;
+import com.google.common.collect.Lists;
+import com.kylinolap.common.KylinConfig;
+import com.kylinolap.common.persistence.ResourceStore;
+import com.kylinolap.common.persistence.RootPersistentEntity;
+import com.kylinolap.cube.CubeDescManager;
+
+@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
+public class CubeInstance extends RootPersistentEntity {
+
+ public static CubeInstance create(String cubeName, String projectName, CubeDesc cubeDesc) {
+ CubeInstance cubeInstance = new CubeInstance();
+
+ cubeInstance.setConfig(cubeDesc.getConfig());
+ cubeInstance.setName(cubeName);
+ cubeInstance.setDescName(cubeDesc.getName());
+ cubeInstance.setCreateTime(formatTime(System.currentTimeMillis()));
+ cubeInstance.setSegments(new ArrayList<CubeSegment>());
+ cubeInstance.setStatus(CubeStatusEnum.DISABLED);
+ cubeInstance.updateRandomUuid();
+
+ return cubeInstance;
+ }
+
+ @JsonIgnore
+ private KylinConfig config;
+ @JsonProperty("name")
+ private String name;
+ @JsonProperty("owner")
+ private String owner;
+ @JsonProperty("version")
+ private String version; // user info only, we don't do version control
+ @JsonProperty("descriptor")
+ private String descName;
+ // Mark cube priority for query
+ @JsonProperty("cost")
+ private int cost = 50;
+ @JsonProperty("status")
+ private CubeStatusEnum status;
+
+ @JsonManagedReference
+ @JsonProperty("segments")
+ private List<CubeSegment> segments = new ArrayList<CubeSegment>();
+
+ @JsonProperty("create_time")
+ private String createTime;
+
+ public List<CubeSegment> getBuildingSegments() {
+ List<CubeSegment> buildingSegments = new ArrayList<CubeSegment>();
+ if (null != segments) {
+ for (CubeSegment segment : segments) {
+ if (CubeSegmentStatusEnum.NEW == segment.getStatus() || CubeSegmentStatusEnum.READY_PENDING == segment.getStatus()) {
+ buildingSegments.add(segment);
+ }
+ }
+ }
+
+ return buildingSegments;
+ }
+
+ public long getAllocatedEndDate() {
+ if (null == segments || segments.size() == 0) {
+ return 0;
+ }
+
+ Collections.sort(segments);
+
+ return segments.get(segments.size() - 1).getDateRangeEnd();
+ }
+
+ public long getAllocatedStartDate() {
+ if (null == segments || segments.size() == 0) {
+ return 0;
+ }
+
+ Collections.sort(segments);
+
+ return segments.get(0).getDateRangeStart();
+ }
+
+ public List<CubeSegment> getMergingSegments() {
+ return this.getMergingSegments(null);
+ }
+
+ public List<CubeSegment> getMergingSegments(CubeSegment cubeSegment) {
+ CubeSegment buildingSegment;
+ if (cubeSegment == null) {
+ List<CubeSegment> buildingSegments = getBuildingSegments();
+ if (buildingSegments.size() == 0) {
+ return Collections.emptyList();
+ }
+ buildingSegment = buildingSegments.get(0);
+ } else {
+ buildingSegment = cubeSegment;
+ }
+
+ List<CubeSegment> mergingSegments = new ArrayList<CubeSegment>();
+ if (null != this.segments) {
+ for (CubeSegment segment : this.segments) {
+ if (segment.getStatus() == CubeSegmentStatusEnum.READY) {
+ if (buildingSegment.getDateRangeStart() <= segment.getDateRangeStart() && buildingSegment.getDateRangeEnd() >= segment.getDateRangeEnd()) {
+ mergingSegments.add(segment);
+ }
+ }
+ }
+ }
+ return mergingSegments;
+
+ }
+
+ public List<CubeSegment> getRebuildingSegments() {
+ List<CubeSegment> buildingSegments = getBuildingSegments();
+ if (buildingSegments.size() == 0) {
+ return Collections.emptyList();
+ } else {
+ List<CubeSegment> rebuildingSegments = new ArrayList<CubeSegment>();
+ if (null != this.segments) {
+ long startDate = buildingSegments.get(0).getDateRangeStart();
+ long endDate = buildingSegments.get(buildingSegments.size() - 1).getDateRangeEnd();
+ for (CubeSegment segment : this.segments) {
+ if (segment.getStatus() == CubeSegmentStatusEnum.READY) {
+ if (startDate >= segment.getDateRangeStart() && startDate < segment.getDateRangeEnd() && segment.getDateRangeEnd() < endDate) {
+ rebuildingSegments.add(segment);
+ continue;
+ }
+ if (startDate <= segment.getDateRangeStart() && endDate >= segment.getDateRangeEnd()) {
+ rebuildingSegments.add(segment);
+ continue;
+ }
+ }
+ }
+ }
+
+ return rebuildingSegments;
+ }
+ }
+
+// public CubeDesc getDescriptor() {
+// return CubeDescManager.getInstance(config).getCubeDesc(descName);
+// }
+
+
+ public boolean isReady() {
+ return getStatus() == CubeStatusEnum.READY;
+ }
+
+ public String getResourcePath() {
+ return concatResourcePath(name);
+ }
+
+ public static String concatResourcePath(String cubeName) {
+ return ResourceStore.CUBE_RESOURCE_ROOT + "/" + cubeName + ".json";
+ }
+
+ @Override
+ public String toString() {
+ return "Cube [name=" + name + "]";
+ }
+
+ // ============================================================================
+
+ @JsonProperty("size_kb")
+ public long getSizeKB() {
+ long sizeKb = 0L;
+
+ for (CubeSegment cubeSegment : this.getSegments(CubeSegmentStatusEnum.READY)) {
+ sizeKb += cubeSegment.getSizeKB();
+ }
+
+ return sizeKb;
+ }
+
+ @JsonProperty("source_records_count")
+ public long getSourceRecordCount() {
+ long sizeRecordCount = 0L;
+
+ for (CubeSegment cubeSegment : this.getSegments(CubeSegmentStatusEnum.READY)) {
+ sizeRecordCount += cubeSegment.getSourceRecords();
+ }
+
+ return sizeRecordCount;
+ }
+
+ @JsonProperty("source_records_size")
+ public long getSourceRecordSize() {
+ long sizeRecordSize = 0L;
+
+ for (CubeSegment cubeSegment : this.getSegments(CubeSegmentStatusEnum.READY)) {
+ sizeRecordSize += cubeSegment.getSourceRecordsSize();
+ }
+
+ return sizeRecordSize;
+ }
+
+ public KylinConfig getConfig() {
+ return config;
+ }
+
+ public void setConfig(KylinConfig config) {
+ this.config = config;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getOwner() {
+ return owner;
+ }
+
+ public void setOwner(String owner) {
+ this.owner = owner;
+ }
+
+ public String getVersion() {
+ return version;
+ }
+
+ public void setVersion(String version) {
+ this.version = version;
+ }
+
+ public String getDescName() {
+ return descName.toUpperCase();
+ }
+
+ public String getOriginDescName() {
+ return descName;
+ }
+
+ public void setDescName(String descName) {
+ this.descName = descName;
+ }
+
+ public int getCost() {
+ return cost;
+ }
+
+ public void setCost(int cost) {
+ this.cost = cost;
+ }
+
+ public CubeStatusEnum getStatus() {
+ return status;
+ }
+
+ public void setStatus(CubeStatusEnum status) {
+ this.status = status;
+ }
+
+ public CubeSegment getFirstSegment() {
+ if (this.segments == null || this.segments.size() == 0) {
+ return null;
+ } else {
+ return this.segments.get(0);
+ }
+ }
+
+ public CubeSegment getLatestReadySegment() {
+ CubeSegment latest = null;
+ for (int i = segments.size() - 1; i >= 0; i--) {
+ CubeSegment seg = segments.get(i);
+ if (seg.getStatus() != CubeSegmentStatusEnum.READY)
+ continue;
+ if (latest == null || latest.getDateRangeEnd() < seg.getDateRangeEnd()) {
+ latest = seg;
+ }
+ }
+ return latest;
+ }
+
+ public List<CubeSegment> getSegments() {
+ return segments;
+ }
+
+ public List<CubeSegment> getSegments(CubeSegmentStatusEnum status) {
+ List<CubeSegment> result = new ArrayList<CubeSegment>();
+
+ for (CubeSegment segment : segments) {
+ if (segment.getStatus() == status) {
+ result.add(segment);
+ }
+ }
+
+ return result;
+ }
+
+ public List<CubeSegment> getSegment(CubeSegmentStatusEnum status) {
+ List<CubeSegment> result = Lists.newArrayList();
+ for (CubeSegment segment : segments) {
+ if (segment.getStatus() == status) {
+ result.add(segment);
+ }
+ }
+ return result;
+ }
+
+ public CubeSegment getSegment(String name, CubeSegmentStatusEnum status) {
+ for (CubeSegment segment : segments) {
+ if ((null != segment.getName() && segment.getName().equals(name)) && segment.getStatus() == status) {
+ return segment;
+ }
+ }
+
+ return null;
+ }
+
+ public void setSegments(List<CubeSegment> segments) {
+ this.segments = segments;
+ }
+
+ public CubeSegment getSegmentById(String segmentId) {
+ for (CubeSegment segment : segments) {
+ if (Objects.equal(segment.getUuid(), segmentId)) {
+ return segment;
+ }
+ }
+ return null;
+ }
+
+ public String getCreateTime() {
+ return createTime;
+ }
+
+ public void setCreateTime(String createTime) {
+ this.createTime = createTime;
+ }
+
+ public long[] getDateRange() {
+ List<CubeSegment> readySegments = getSegment(CubeSegmentStatusEnum.READY);
+ if (readySegments.isEmpty()) {
+ return new long[]{0L, 0L};
+ }
+ long start = Long.MAX_VALUE;
+ long end = Long.MIN_VALUE;
+ for (CubeSegment segment : readySegments) {
+ if (segment.getDateRangeStart() < start) {
+ start = segment.getDateRangeStart();
+ }
+ if (segment.getDateRangeEnd() > end) {
+ end = segment.getDateRangeEnd();
+ }
+ }
+ return new long[]{start, end};
+ }
+
+// private boolean appendOnHll() {
+// CubePartitionDesc cubePartitionDesc = getDescriptor().getCubePartitionDesc();
+// if (cubePartitionDesc == null) {
+// return false;
+// }
+// if (cubePartitionDesc.getPartitionDateColumn() == null) {
+// return false;
+// }
+// if (cubePartitionDesc.getCubePartitionType() != CubePartitionDesc.CubePartitionType.APPEND) {
+// return false;
+// }
+// return getDescriptor().hasHolisticCountDistinctMeasures();
+// }
+
+// public boolean appendBuildOnHllMeasure(long startDate, long endDate) {
+// if (!appendOnHll()) {
+// return false;
+// }
+// List<CubeSegment> readySegments = getSegment(CubeSegmentStatusEnum.READY);
+// if (readySegments.isEmpty()) {
+// return false;
+// }
+// for (CubeSegment readySegment: readySegments) {
+// if (readySegment.getDateRangeStart() == startDate && readySegment.getDateRangeEnd() == endDate) {
+// //refresh build
+// return false;
+// }
+// }
+// return true;
+// }
+
+// public boolean needMergeImmediatelyAfterBuild(CubeSegment segment) {
+// if (!appendOnHll()) {
+// return false;
+// }
+// List<CubeSegment> readySegments = getSegment(CubeSegmentStatusEnum.READY);
+// if (readySegments.isEmpty()) {
+// return false;
+// }
+// for (CubeSegment readySegment: readySegments) {
+// if (readySegment.getDateRangeEnd() > segment.getDateRangeStart()) {
+// //has overlap and not refresh
+// return true;
+// }
+// }
+// return false;
+// }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/be56ebff/cube/src/main/java/com/kylinolap/cube/model/v1/CubeSegment.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/model/v1/CubeSegment.java b/cube/src/main/java/com/kylinolap/cube/model/v1/CubeSegment.java
new file mode 100644
index 0000000..3919685
--- /dev/null
+++ b/cube/src/main/java/com/kylinolap/cube/model/v1/CubeSegment.java
@@ -0,0 +1,306 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.cube.model.v1;
+
+import java.text.SimpleDateFormat;
+import java.util.Collection;
+import java.util.Map;
+import java.util.TimeZone;
+import java.util.concurrent.ConcurrentHashMap;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonBackReference;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Objects;
+import com.kylinolap.metadata.model.TblColRef;
+
+@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
+public class CubeSegment implements Comparable<CubeSegment> {
+
+ @JsonBackReference
+ private CubeInstance cubeInstance;
+ @JsonProperty("uuid")
+ private String uuid;
+ @JsonProperty("name")
+ private String name;
+ @JsonProperty("storage_location_identifier")
+ private String storageLocationIdentifier; // HTable name
+ @JsonProperty("date_range_start")
+ private long dateRangeStart;
+ @JsonProperty("date_range_end")
+ private long dateRangeEnd;
+ @JsonProperty("status")
+ private CubeSegmentStatusEnum status;
+ @JsonProperty("size_kb")
+ private long sizeKB;
+ @JsonProperty("source_records")
+ private long sourceRecords;
+ @JsonProperty("source_records_size")
+ private long sourceRecordsSize;
+ @JsonProperty("last_build_time")
+ private long lastBuildTime;
+ @JsonProperty("last_build_job_id")
+ private String lastBuildJobID;
+ @JsonProperty("create_time")
+ private String createTime;
+
+ @JsonProperty("binary_signature")
+ private String binarySignature; // a hash of cube schema and dictionary ID,
+ // used for sanity check
+
+ @JsonProperty("dictionaries")
+ private ConcurrentHashMap<String, String> dictionaries; // table/column ==> dictionary resource path
+ @JsonProperty("snapshots")
+ private ConcurrentHashMap<String, String> snapshots; // table name ==> snapshot resource path
+
+// public CubeDesc getCubeDesc() {
+// return getCubeInstance().getDescriptor();
+// }
+
+ /**
+ * @param startDate
+ * @param endDate
+ * @return if(startDate == 0 && endDate == 0), returns "FULL_BUILD", else
+ * returns "yyyyMMddHHmmss_yyyyMMddHHmmss"
+ */
+ public static String getSegmentName(long startDate, long endDate) {
+ if (startDate == 0 && endDate == 0) {
+ return "FULL_BUILD";
+ }
+
+ SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMddHHmmss");
+ dateFormat.setTimeZone(TimeZone.getTimeZone("GMT"));
+
+ return dateFormat.format(startDate) + "_" + dateFormat.format(endDate);
+ }
+
+ // ============================================================================
+
+
+ public String getUuid() {
+ return uuid;
+ }
+
+ public void setUuid(String id) {
+ this.uuid = id;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public long getDateRangeStart() {
+ return dateRangeStart;
+ }
+
+ public void setDateRangeStart(long dateRangeStart) {
+ this.dateRangeStart = dateRangeStart;
+ }
+
+ public long getDateRangeEnd() {
+ return dateRangeEnd;
+ }
+
+ public void setDateRangeEnd(long dateRangeEnd) {
+ this.dateRangeEnd = dateRangeEnd;
+ }
+
+ public CubeSegmentStatusEnum getStatus() {
+ return status;
+ }
+
+ public void setStatus(CubeSegmentStatusEnum status) {
+ this.status = status;
+ }
+
+ public long getSizeKB() {
+ return sizeKB;
+ }
+
+ public void setSizeKB(long sizeKB) {
+ this.sizeKB = sizeKB;
+ }
+
+ public long getSourceRecords() {
+ return sourceRecords;
+ }
+
+ public void setSourceRecords(long sourceRecords) {
+ this.sourceRecords = sourceRecords;
+ }
+
+ public long getSourceRecordsSize() {
+ return sourceRecordsSize;
+ }
+
+ public void setSourceRecordsSize(long sourceRecordsSize) {
+ this.sourceRecordsSize = sourceRecordsSize;
+ }
+
+ public long getLastBuildTime() {
+ return lastBuildTime;
+ }
+
+ public void setLastBuildTime(long lastBuildTime) {
+ this.lastBuildTime = lastBuildTime;
+ }
+
+ public String getLastBuildJobID() {
+ return lastBuildJobID;
+ }
+
+ public void setLastBuildJobID(String lastBuildJobID) {
+ this.lastBuildJobID = lastBuildJobID;
+ }
+
+ public String getCreateTime() {
+ return createTime;
+ }
+
+ public void setCreateTime(String createTime) {
+ this.createTime = createTime;
+ }
+
+ public String getBinarySignature() {
+ return binarySignature;
+ }
+
+ public void setBinarySignature(String binarySignature) {
+ this.binarySignature = binarySignature;
+ }
+
+ public CubeInstance getCubeInstance() {
+ return cubeInstance;
+ }
+
+ public void setCubeInstance(CubeInstance cubeInstance) {
+ this.cubeInstance = cubeInstance;
+ }
+
+ public String getStorageLocationIdentifier() {
+
+ return storageLocationIdentifier;
+ }
+
+ public Map<String, String> getDictionaries() {
+ if (dictionaries == null)
+ dictionaries = new ConcurrentHashMap<String, String>();
+ return dictionaries;
+ }
+
+ public Map<String, String> getSnapshots() {
+ if (snapshots == null)
+ snapshots = new ConcurrentHashMap<String, String>();
+ return snapshots;
+ }
+
+ public String getSnapshotResPath(String table) {
+ return getSnapshots().get(table);
+ }
+
+ public void putSnapshotResPath(String table, String snapshotResPath) {
+ getSnapshots().put(table, snapshotResPath);
+ }
+
+ public Collection<String> getDictionaryPaths() {
+ return getDictionaries().values();
+ }
+
+ public Collection<String> getSnapshotPaths() {
+ return getSnapshots().values();
+ }
+
+ public String getDictResPath(TblColRef col) {
+ return getDictionaries().get(dictKey(col));
+ }
+
+ public void putDictResPath(TblColRef col, String dictResPath) {
+ getDictionaries().put(dictKey(col), dictResPath);
+ }
+
+ private String dictKey(TblColRef col) {
+ return col.getTable() + "/" + col.getName();
+ }
+
+ /**
+ * @param storageLocationIdentifier
+ * the storageLocationIdentifier to set
+ */
+ public void setStorageLocationIdentifier(String storageLocationIdentifier) {
+ this.storageLocationIdentifier = storageLocationIdentifier;
+ }
+
+ @Override
+ public int compareTo(CubeSegment other) {
+ if (this.dateRangeEnd < other.dateRangeEnd) {
+ return -1;
+ } else if (this.dateRangeEnd > other.dateRangeEnd) {
+ return 1;
+ } else {
+ return 0;
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((cubeInstance == null) ? 0 : cubeInstance.hashCode());
+ result = prime * result + ((name == null) ? 0 : name.hashCode());
+ result = prime * result + ((status == null) ? 0 : status.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ CubeSegment other = (CubeSegment) obj;
+ if (cubeInstance == null) {
+ if (other.cubeInstance != null)
+ return false;
+ } else if (!cubeInstance.equals(other.cubeInstance))
+ return false;
+ if (name == null) {
+ if (other.name != null)
+ return false;
+ } else if (!name.equals(other.name))
+ return false;
+ if (status != other.status)
+ return false;
+ return true;
+ }
+
+ @Override
+ public String toString() {
+ return Objects.toStringHelper(this)
+ .add("uuid", uuid)
+ .add("create_time:", createTime)
+ .add("name", name)
+ .add("last_build_job_id", lastBuildJobID)
+ .add("status", status)
+ .toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/be56ebff/cube/src/main/java/com/kylinolap/cube/model/v1/CubeSegmentStatusEnum.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/model/v1/CubeSegmentStatusEnum.java b/cube/src/main/java/com/kylinolap/cube/model/v1/CubeSegmentStatusEnum.java
new file mode 100644
index 0000000..04cd57c
--- /dev/null
+++ b/cube/src/main/java/com/kylinolap/cube/model/v1/CubeSegmentStatusEnum.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.cube.model.v1;
+
+/**
+ * @author xduo
+ *
+ */
+public enum CubeSegmentStatusEnum {
+ NEW, READY, READY_PENDING
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/be56ebff/cube/src/main/java/com/kylinolap/cube/model/v1/CubeSegmentTypeEnum.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/model/v1/CubeSegmentTypeEnum.java b/cube/src/main/java/com/kylinolap/cube/model/v1/CubeSegmentTypeEnum.java
new file mode 100644
index 0000000..f339e65
--- /dev/null
+++ b/cube/src/main/java/com/kylinolap/cube/model/v1/CubeSegmentTypeEnum.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.cube.model.v1;
+
+/**
+ * @author ysong1
+ *
+ */
+public enum CubeSegmentTypeEnum {
+ TRANSIENT, PERMANENT
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/be56ebff/cube/src/main/java/com/kylinolap/cube/model/v1/CubeStatusEnum.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/model/v1/CubeStatusEnum.java b/cube/src/main/java/com/kylinolap/cube/model/v1/CubeStatusEnum.java
new file mode 100644
index 0000000..77afb16
--- /dev/null
+++ b/cube/src/main/java/com/kylinolap/cube/model/v1/CubeStatusEnum.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.cube.model.v1;
+
+public enum CubeStatusEnum {
+
+ DISABLED, BUILDING, READY, DESCBROKEN
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/be56ebff/cube/src/main/java/com/kylinolap/cube/model/v1/ProjectInstance.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/model/v1/ProjectInstance.java b/cube/src/main/java/com/kylinolap/cube/model/v1/ProjectInstance.java
index e5f9fae..6bfda84 100644
--- a/cube/src/main/java/com/kylinolap/cube/model/v1/ProjectInstance.java
+++ b/cube/src/main/java/com/kylinolap/cube/model/v1/ProjectInstance.java
@@ -120,10 +120,6 @@ public class ProjectInstance extends RootPersistentEntity {
this.status = status;
}
- /**
- * @deprecated use createTimeUTC instead
- * @return
- */
public String getCreateTime() {
return createTime;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/be56ebff/cube/src/test/java/com/kylinolap/cube/SegmentManagementTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/com/kylinolap/cube/SegmentManagementTest.java b/cube/src/test/java/com/kylinolap/cube/SegmentManagementTest.java
index 2121488..953aac6 100644
--- a/cube/src/test/java/com/kylinolap/cube/SegmentManagementTest.java
+++ b/cube/src/test/java/com/kylinolap/cube/SegmentManagementTest.java
@@ -207,8 +207,8 @@ public class SegmentManagementTest extends LocalFileMetadataTestCase {
assertEquals(1384240200000L, cubeInstance.getAllocatedStartDate());
assertEquals(1386835200000L, cubeInstance.getAllocatedEndDate());
assertEquals("job_4", cubeInstance.getSegments().get(0).getLastBuildJobID());
- assertEquals(20000L, cubeInstance.getSegments().get(0).getSourceRecords());
- assertEquals(1216024L, cubeInstance.getSegments().get(0).getSourceRecordsSize());
+ assertEquals(20000L, cubeInstance.getSegments().get(0).getInputRecords());
+ assertEquals(1216024L, cubeInstance.getSegments().get(0).getInputRecordsSize());
System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance));
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/be56ebff/invertedindex/src/main/java/com/kylinolap/invertedindex/IIInstance.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/com/kylinolap/invertedindex/IIInstance.java b/invertedindex/src/main/java/com/kylinolap/invertedindex/IIInstance.java
index c0c362a..7a6f1fc 100644
--- a/invertedindex/src/main/java/com/kylinolap/invertedindex/IIInstance.java
+++ b/invertedindex/src/main/java/com/kylinolap/invertedindex/IIInstance.java
@@ -202,23 +202,23 @@ public class IIInstance extends RootPersistentEntity implements IRealization {
return sizeKb;
}
- @JsonProperty("source_records_count")
- public long getSourceRecordCount() {
+ @JsonProperty("input_records_count")
+ public long getInputRecordCount() {
long sizeRecordCount = 0L;
for (IISegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) {
- sizeRecordCount += cubeSegment.getSourceRecords();
+ sizeRecordCount += cubeSegment.getInputRecords();
}
return sizeRecordCount;
}
- @JsonProperty("source_records_size")
- public long getSourceRecordSize() {
+ @JsonProperty("input_records_size")
+ public long getInputRecordSize() {
long sizeRecordSize = 0L;
for (IISegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) {
- sizeRecordSize += cubeSegment.getSourceRecordsSize();
+ sizeRecordSize += cubeSegment.getInputRecordsSize();
}
return sizeRecordSize;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/be56ebff/invertedindex/src/main/java/com/kylinolap/invertedindex/IISegment.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/com/kylinolap/invertedindex/IISegment.java b/invertedindex/src/main/java/com/kylinolap/invertedindex/IISegment.java
index 3416668..0e15fcf 100644
--- a/invertedindex/src/main/java/com/kylinolap/invertedindex/IISegment.java
+++ b/invertedindex/src/main/java/com/kylinolap/invertedindex/IISegment.java
@@ -58,10 +58,10 @@ public class IISegment implements Comparable<IISegment>, ISegment {
private SegmentStatusEnum status;
@JsonProperty("size_kb")
private long sizeKB;
- @JsonProperty("source_records")
- private long sourceRecords;
- @JsonProperty("source_records_size")
- private long sourceRecordsSize;
+ @JsonProperty("input_records")
+ private long inputRecords;
+ @JsonProperty("input_records_size")
+ private long inputRecordsSize;
@JsonProperty("last_build_time")
private long lastBuildTime;
@JsonProperty("last_build_job_id")
@@ -152,20 +152,20 @@ public class IISegment implements Comparable<IISegment>, ISegment {
this.sizeKB = sizeKB;
}
- public long getSourceRecords() {
- return sourceRecords;
+ public long getInputRecords() {
+ return inputRecords;
}
- public void setSourceRecords(long sourceRecords) {
- this.sourceRecords = sourceRecords;
+ public void setInputRecords(long inputRecords) {
+ this.inputRecords = inputRecords;
}
- public long getSourceRecordsSize() {
- return sourceRecordsSize;
+ public long getInputRecordsSize() {
+ return inputRecordsSize;
}
- public void setSourceRecordsSize(long sourceRecordsSize) {
- this.sourceRecordsSize = sourceRecordsSize;
+ public void setInputRecordsSize(long inputRecordsSize) {
+ this.inputRecordsSize = inputRecordsSize;
}
public long getLastBuildTime() {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/be56ebff/job/src/main/java/com/kylinolap/job/CubeMetadataUpgrade.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/com/kylinolap/job/CubeMetadataUpgrade.java b/job/src/main/java/com/kylinolap/job/CubeMetadataUpgrade.java
index 53277ef..1ea0317 100644
--- a/job/src/main/java/com/kylinolap/job/CubeMetadataUpgrade.java
+++ b/job/src/main/java/com/kylinolap/job/CubeMetadataUpgrade.java
@@ -8,11 +8,29 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
+import com.kylinolap.common.KylinConfig;
+import com.kylinolap.common.persistence.JsonSerializer;
+import com.kylinolap.common.persistence.ResourceStore;
+import com.kylinolap.common.persistence.RootPersistentEntity;
+import com.kylinolap.common.util.JsonUtil;
+import com.kylinolap.cube.CubeDescManager;
+import com.kylinolap.cube.CubeDescUpgrader;
+import com.kylinolap.cube.CubeManager;
import com.kylinolap.cube.model.CubeDesc;
+import com.kylinolap.cube.model.v1.CubeSegmentStatusEnum;
+import com.kylinolap.cube.model.v1.CubeStatusEnum;
import com.kylinolap.job.common.HadoopShellExecutable;
import com.kylinolap.job.common.MapReduceExecutable;
import com.kylinolap.job.common.ShellExecutable;
@@ -20,32 +38,26 @@ import com.kylinolap.job.constant.ExecutableConstants;
import com.kylinolap.job.constant.JobStatusEnum;
import com.kylinolap.job.constant.JobStepStatusEnum;
import com.kylinolap.job.cube.CubingJob;
+import com.kylinolap.job.execution.AbstractExecutable;
import com.kylinolap.job.execution.ExecutableState;
-import com.kylinolap.job.hadoop.cube.*;
+import com.kylinolap.job.hadoop.cube.BaseCuboidJob;
+import com.kylinolap.job.hadoop.cube.CubeHFileJob;
+import com.kylinolap.job.hadoop.cube.FactDistinctColumnsJob;
+import com.kylinolap.job.hadoop.cube.MergeCuboidJob;
+import com.kylinolap.job.hadoop.cube.NDCuboidJob;
+import com.kylinolap.job.hadoop.cube.RangeKeyDistributionJob;
import com.kylinolap.job.hadoop.dict.CreateDictionaryJob;
import com.kylinolap.job.hadoop.hbase.BulkLoadJob;
import com.kylinolap.job.hadoop.hbase.CreateHTableJob;
-import com.kylinolap.job.execution.AbstractExecutable;
import com.kylinolap.job.manager.ExecutableManager;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-import com.kylinolap.common.KylinConfig;
-import com.kylinolap.common.persistence.JsonSerializer;
-import com.kylinolap.common.persistence.ResourceStore;
-import com.kylinolap.common.util.JsonUtil;
-import com.kylinolap.cube.CubeDescManager;
-import com.kylinolap.cube.CubeDescUpgrader;
import com.kylinolap.metadata.MetadataConstances;
import com.kylinolap.metadata.MetadataManager;
+import com.kylinolap.metadata.model.SegmentStatusEnum;
import com.kylinolap.metadata.model.TableDesc;
import com.kylinolap.metadata.project.ProjectInstance;
import com.kylinolap.metadata.project.ProjectManager;
import com.kylinolap.metadata.project.RealizationEntry;
+import com.kylinolap.metadata.realization.RealizationStatusEnum;
import com.kylinolap.metadata.realization.RealizationType;
/**
@@ -80,6 +92,7 @@ public class CubeMetadataUpgrade {
upgradeTableDesceExd();
upgradeCubeDesc();
upgradeProjectInstance();
+ upgradeCubeInstance();
upgradeJobInstance();
verify();
@@ -125,7 +138,7 @@ public class CubeMetadataUpgrade {
}
}
-
+
private void upgradeTableDesc() {
List<String> paths = listResourceStore(ResourceStore.TABLE_RESOURCE_ROOT);
for (String path : paths) {
@@ -238,7 +251,7 @@ public class CubeMetadataUpgrade {
newPrj.setDescription(oldPrj.getDescription());
newPrj.setLastModified(oldPrj.getLastModified());
newPrj.setLastUpdateTime(oldPrj.getLastUpdateTime());
- newPrj.setCreateTime(oldPrj.getCreateTime());
+ newPrj.setCreateTimeUTC(RootPersistentEntity.parseTime(oldPrj.getCreateTime()));
newPrj.setStatus(oldPrj.getStatus());
List<RealizationEntry> realizationEntries = Lists.newArrayList();
for (String cube : oldPrj.getCubes()) {
@@ -265,6 +278,75 @@ public class CubeMetadataUpgrade {
}
}
+
+ private void upgradeCubeInstance() {
+
+ ResourceStore store = getStore();
+ List<String> paths = listResourceStore(ResourceStore.CUBE_RESOURCE_ROOT);
+ for (String path : paths) {
+
+ com.kylinolap.cube.model.v1.CubeInstance cubeInstance = null;
+ try {
+ cubeInstance = store.getResource(path, com.kylinolap.cube.model.v1.CubeInstance.class, new JsonSerializer<com.kylinolap.cube.model.v1.CubeInstance>(com.kylinolap.cube.model.v1.CubeInstance.class));
+ cubeInstance.setConfig(config);
+
+ com.kylinolap.cube.CubeInstance newInstance = new com.kylinolap.cube.CubeInstance();
+ newInstance.setName(cubeInstance.getName());
+ newInstance.setDescName(cubeInstance.getDescName());
+ newInstance.setOwner(cubeInstance.getOwner());
+ newInstance.setUuid(cubeInstance.getUuid());
+ newInstance.setVersion(cubeInstance.getVersion());
+ newInstance.setCreateTimeUTC(RootPersistentEntity.parseTime(cubeInstance.getCreateTime()));
+
+ //status
+ if(cubeInstance.getStatus() == CubeStatusEnum.BUILDING) {
+ newInstance.setStatus(RealizationStatusEnum.BUILDING);
+ } else if(cubeInstance.getStatus() == CubeStatusEnum.DESCBROKEN) {
+ newInstance.setStatus(RealizationStatusEnum.DESCBROKEN);
+ } else if(cubeInstance.getStatus() == CubeStatusEnum.DISABLED) {
+ newInstance.setStatus(RealizationStatusEnum.DISABLED);
+ } else if(cubeInstance.getStatus() == CubeStatusEnum.READY) {
+ newInstance.setStatus(RealizationStatusEnum.READY);
+ }
+
+ List<com.kylinolap.cube.CubeSegment> newSegments = Lists.newArrayList();
+ // segment
+ for (com.kylinolap.cube.model.v1.CubeSegment segment : cubeInstance.getSegments()) {
+ com.kylinolap.cube.CubeSegment newSeg = new com.kylinolap.cube.CubeSegment();
+ newSegments.add(newSeg);
+
+ newSeg.setUuid(segment.getUuid());
+ newSeg.setName(segment.getName());
+ newSeg.setStorageLocationIdentifier(segment.getStorageLocationIdentifier());
+ newSeg.setDateRangeStart(segment.getDateRangeStart());
+ newSeg.setDateRangeEnd(segment.getDateRangeEnd());
+
+ if(segment.getStatus() == CubeSegmentStatusEnum.NEW) {
+ newSeg.setStatus(SegmentStatusEnum.NEW);
+ } else if(segment.getStatus() == CubeSegmentStatusEnum.READY) {
+ newSeg.setStatus(SegmentStatusEnum.READY);
+ } else if(segment.getStatus() == CubeSegmentStatusEnum.READY_PENDING) {
+ newSeg.setStatus(SegmentStatusEnum.READY_PENDING);
+ }
+
+ newSeg.setSizeKB(segment.getSizeKB());
+ newSeg.setInputRecords(segment.getSourceRecords());
+ newSeg.setInputRecordsSize(segment.getSourceRecordsSize());
+ newSeg.setLastBuildTime(segment.getLastBuildTime());
+ newSeg.setLastBuildJobID(segment.getLastBuildJobID());
+ newSeg.setCreateTimeUTC(RootPersistentEntity.parseTime(segment.getCreateTime()));
+ newSeg.setBinarySignature(segment.getBinarySignature());
+ newSeg.setDictionaries((ConcurrentHashMap<String, String>)segment.getDictionaries());
+ newSeg.setSnapshots((ConcurrentHashMap<String, String>)segment.getSnapshots());
+ }
+
+ newInstance.setSegments(newSegments);
+ store.putResource(newInstance.getResourcePath(), newInstance, CubeManager.CUBE_SERIALIZER);
+ } catch (Exception e) {
+ logger.error("Error during load cube instance " + path, e);
+ }
+ }
+ }
private MetadataManager getMetadataManager() {
return MetadataManager.getInstance(config);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/be56ebff/job/src/main/java/com/kylinolap/job/cube/UpdateCubeInfoAfterBuildStep.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/com/kylinolap/job/cube/UpdateCubeInfoAfterBuildStep.java b/job/src/main/java/com/kylinolap/job/cube/UpdateCubeInfoAfterBuildStep.java
index 1366386..ac9f997 100644
--- a/job/src/main/java/com/kylinolap/job/cube/UpdateCubeInfoAfterBuildStep.java
+++ b/job/src/main/java/com/kylinolap/job/cube/UpdateCubeInfoAfterBuildStep.java
@@ -106,8 +106,8 @@ public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
segment.setLastBuildJobID(getCubingJobId());
segment.setLastBuildTime(System.currentTimeMillis());
segment.setSizeKB(size);
- segment.setSourceRecords(sourceCount);
- segment.setSourceRecordsSize(sourceSize);
+ segment.setInputRecords(sourceCount);
+ segment.setInputRecordsSize(sourceSize);
try {
if (segmentReady) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/be56ebff/job/src/main/java/com/kylinolap/job/cube/UpdateCubeInfoAfterMergeStep.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/com/kylinolap/job/cube/UpdateCubeInfoAfterMergeStep.java b/job/src/main/java/com/kylinolap/job/cube/UpdateCubeInfoAfterMergeStep.java
index ef8f889..cc1d1ad 100644
--- a/job/src/main/java/com/kylinolap/job/cube/UpdateCubeInfoAfterMergeStep.java
+++ b/job/src/main/java/com/kylinolap/job/cube/UpdateCubeInfoAfterMergeStep.java
@@ -57,14 +57,14 @@ public class UpdateCubeInfoAfterMergeStep extends AbstractExecutable {
long sourceSize = 0L;
for (String id : mergingSegmentIds) {
CubeSegment segment = cube.getSegmentById(id);
- sourceCount += segment.getSourceRecords();
- sourceSize += segment.getSourceRecordsSize();
+ sourceCount += segment.getInputRecords();
+ sourceSize += segment.getInputRecordsSize();
}
// update segment info
mergedSegment.setSizeKB(cubeSize);
- mergedSegment.setSourceRecords(sourceCount);
- mergedSegment.setSourceRecordsSize(sourceSize);
+ mergedSegment.setInputRecords(sourceCount);
+ mergedSegment.setInputRecordsSize(sourceSize);
mergedSegment.setLastBuildJobID(getCubingJobId());
mergedSegment.setLastBuildTime(System.currentTimeMillis());
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/be56ebff/metadata/src/main/java/com/kylinolap/metadata/project/ProjectInstance.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/project/ProjectInstance.java b/metadata/src/main/java/com/kylinolap/metadata/project/ProjectInstance.java
index 910d9bb..fa0015a 100644
--- a/metadata/src/main/java/com/kylinolap/metadata/project/ProjectInstance.java
+++ b/metadata/src/main/java/com/kylinolap/metadata/project/ProjectInstance.java
@@ -52,9 +52,6 @@ public class ProjectInstance extends RootPersistentEntity {
@JsonProperty("status")
private ProjectStatusEnum status;
- @JsonProperty("create_time")
- private String createTime;
-
@JsonProperty("create_time_utc")
private long createTimeUTC;
@@ -121,23 +118,7 @@ public class ProjectInstance extends RootPersistentEntity {
this.status = status;
}
- /**
- * @deprecated use createTimeUTC instead
- * @return
- */
- public String getCreateTime() {
- return createTime;
- }
-
- public void setCreateTime(String createTime) {
- this.createTime = createTime;
- }
-
public long getCreateTimeUTC() {
- if(createTimeUTC ==0 && createTime !=null) {
- createTimeUTC = parseTime(createTime);
- }
-
return createTimeUTC;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/be56ebff/metadata/src/main/java/com/kylinolap/metadata/project/ProjectManager.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/project/ProjectManager.java b/metadata/src/main/java/com/kylinolap/metadata/project/ProjectManager.java
index c436087..20912e6 100644
--- a/metadata/src/main/java/com/kylinolap/metadata/project/ProjectManager.java
+++ b/metadata/src/main/java/com/kylinolap/metadata/project/ProjectManager.java
@@ -194,7 +194,7 @@ public class ProjectManager {
if (!project.getName().equals(newName)) {
ProjectInstance newProject = this.createProject(newName, project.getOwner(), newDesc);
// FIXME table lost??
- newProject.setCreateTime(project.getCreateTime());
+ newProject.setCreateTimeUTC(project.getCreateTimeUTC());
newProject.recordUpdateTime(System.currentTimeMillis());
newProject.setRealizationEntries(project.getRealizationEntries());
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/be56ebff/server/src/main/java/com/kylinolap/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/com/kylinolap/rest/service/CubeService.java b/server/src/main/java/com/kylinolap/rest/service/CubeService.java
index f6d76b4..45ccb54 100644
--- a/server/src/main/java/com/kylinolap/rest/service/CubeService.java
+++ b/server/src/main/java/com/kylinolap/rest/service/CubeService.java
@@ -390,7 +390,6 @@ public class CubeService extends BasicService {
}
public MetricsResponse calculateMetrics(MetricsRequest request) {
- DateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss z");
List<CubeInstance> cubes = this.getCubeManager().listAllCubes();
MetricsResponse metrics = new MetricsResponse();
Date startTime = (null == request.getStartTime()) ? new Date(-1) : request.getStartTime();
@@ -400,11 +399,7 @@ public class CubeService extends BasicService {
for (CubeInstance cube : cubes) {
Date createdDate = new Date(-1);
- try {
- createdDate = (null == cube.getCreateTime()) ? createdDate : format.parse(cube.getCreateTime());
- } catch (ParseException e) {
- logger.error("", e);
- }
+ createdDate = (cube.getCreateTimeUTC() == 0) ? createdDate : new Date(cube.getCreateTimeUTC());
if (createdDate.getTime() > startTime.getTime() && createdDate.getTime() < endTime.getTime()) {
metrics.increase("totalCubes");