You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/04/26 07:49:48 UTC
[12/14] kylin git commit: minior, refine TableExtDesc, JoinedFlatTable,
CubeJoinedFlatTableDesc
minior, refine TableExtDesc,JoinedFlatTable,CubeJoinedFlatTableDesc
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0762c4dd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0762c4dd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0762c4dd
Branch: refs/heads/master-hadoop3.0
Commit: 0762c4ddbc27be4f471882394e580d94bc836fb5
Parents: a862603
Author: Cheng Wang <ch...@kyligence.io>
Authored: Thu Apr 20 15:09:33 2017 +0800
Committer: liyang-gmt8 <li...@apache.org>
Committed: Tue Apr 25 16:13:53 2017 +0800
----------------------------------------------------------------------
.../cube/model/CubeJoinedFlatTableDesc.java | 2 +-
.../org/apache/kylin/job/JoinedFlatTable.java | 8 +++-
.../kylin/metadata/model/TableExtDesc.java | 44 +++++++++++++++++---
3 files changed, 46 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/0762c4dd/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
index f607466..c49d37a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
@@ -44,7 +44,7 @@ public class CubeJoinedFlatTableDesc implements IJoinedFlatTableDesc, java.io.Se
private int columnCount = 0;
private List<TblColRef> columnList = Lists.newArrayList();
- private Map<TblColRef, Integer> columnIndexMap = Maps.newHashMap();;
+ private Map<TblColRef, Integer> columnIndexMap = Maps.newHashMap();
public CubeJoinedFlatTableDesc(CubeDesc cubeDesc) {
this(cubeDesc, null);
http://git-wip-us.apache.org/repos/asf/kylin/blob/0762c4dd/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 54c1e61..a9da822 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -107,7 +107,13 @@ public class JoinedFlatTable {
}
public static String generateInsertDataStatement(IJoinedFlatTableDesc flatDesc) {
- final KylinConfig kylinConfig = ((CubeSegment) flatDesc.getSegment()).getConfig();
+ CubeSegment segment = ((CubeSegment) flatDesc.getSegment());
+ KylinConfig kylinConfig;
+ if (null == segment) {
+ kylinConfig = KylinConfig.getInstanceFromEnv();
+ } else {
+ kylinConfig = ((CubeSegment) flatDesc.getSegment()).getConfig();
+ }
if (kylinConfig.isAdvancedFlatTableUsed()) {
try {
Class advancedFlatTable = Class.forName(kylinConfig.getAdvancedFlatTableClass());
http://git-wip-us.apache.org/repos/asf/kylin/blob/0762c4dd/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
index 18fefc2..9692f5a 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
@@ -49,9 +49,11 @@ public class TableExtDesc extends RootPersistentEntity {
private List<String[]> sampleRows = new ArrayList<>();
@JsonProperty("last_modified_time")
- private String lastModifiedTime;
+ private long lastModifiedTime;
@JsonProperty("total_rows")
- private String totalRows;
+ private long totalRows;
+ @JsonProperty("mapper_rows")
+ private List<Long> mapRecords = new ArrayList<>();
@JsonProperty("data_source_properties")
private Map<String, String> dataSourceProps = new HashMap<>();
@@ -90,6 +92,14 @@ public class TableExtDesc extends RootPersistentEntity {
return this.sampleRows;
}
+ public void setMapRecords(List<Long> mapRecords) {
+ this.mapRecords = mapRecords;
+ }
+
+ public List<Long> getMapRecords() {
+ return this.mapRecords;
+ }
+
public String getCardinality() {
StringBuffer cardinality = new StringBuffer();
@@ -131,11 +141,11 @@ public class TableExtDesc extends RootPersistentEntity {
this.columnStats = columnStats;
}
- public void setTotalRows(String totalRows) {
+ public void setTotalRows(long totalRows) {
this.totalRows = totalRows;
}
- public String getTotalRows() {
+ public long getTotalRows() {
return this.totalRows;
}
@@ -152,11 +162,11 @@ public class TableExtDesc extends RootPersistentEntity {
this.tableName = this.tableName.toUpperCase();
}
- public void setLastModifiedTime(String lastModifiedTime) {
+ public void setLastModifiedTime(long lastModifiedTime) {
this.lastModifiedTime = lastModifiedTime;
}
- public String getLastModifiedTime() {
+ public long getLastModifiedTime() {
return this.lastModifiedTime;
}
@@ -187,6 +197,9 @@ public class TableExtDesc extends RootPersistentEntity {
@JsonBackReference
private TableExtDesc tableExtDesc;
+ @JsonProperty("column_name")
+ private String columnName;
+
@JsonProperty("max_value")
private String maxValue;
@@ -205,6 +218,9 @@ public class TableExtDesc extends RootPersistentEntity {
@JsonProperty("cardinality")
private long cardinality;
+ @JsonProperty("data_skew_samples")
+ private Map<String, Long> dataSkewSamples = new HashMap<>();
+
@Override
public int compareTo(ColumnStats o) {
return 0;
@@ -213,6 +229,14 @@ public class TableExtDesc extends RootPersistentEntity {
public ColumnStats() {
}
+ public void setColumnName(String columnName) {
+ this.columnName = columnName;
+ }
+
+ public String getColumnName() {
+ return this.columnName;
+ }
+
public void setMaxValue(String maxValue) {
this.maxValue = maxValue;
}
@@ -253,6 +277,14 @@ public class TableExtDesc extends RootPersistentEntity {
return this.cardinality;
}
+ public void setDataSkewSamples(Map<String, Long> dataSkewSamples) {
+ this.dataSkewSamples = dataSkewSamples;
+ }
+
+ public Map<String, Long> getDataSkewSamples() {
+ return this.dataSkewSamples;
+ }
+
public void setColumnSamples(String max, String min, String maxLenValue, String minLenValue) {
this.maxValue = max;
this.minValue = min;