You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/10/08 10:44:46 UTC
[1/2] kylin git commit: KYLIN 2901 Update correct cardinality for
empty table
Repository: kylin
Updated Branches:
refs/heads/master 7a2510d44 -> 0297d9abf
KYLIN 2901 Update correct cardinality for empty table
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a118cae1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a118cae1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a118cae1
Branch: refs/heads/master
Commit: a118cae1c9e0289c50ea42f4d21e93b0d6a0fc11
Parents: 7a2510d
Author: peng.jianhua <pe...@zte.com.cn>
Authored: Mon Sep 25 17:15:00 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sun Oct 8 18:38:50 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/metadata/model/TableExtDesc.java | 13 ++++++++++++-
.../cardinality/HiveColumnCardinalityUpdateJob.java | 4 ++++
2 files changed, 16 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a118cae1/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
index b2a3e06..e4cda8f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
@@ -138,7 +138,18 @@ public class TableExtDesc extends RootPersistentEntity {
}
return cardinality.toString();
}
-
+
+ public String resetCardinality(){
+ StringBuffer cardinality = new StringBuffer();
+ int columnSize=this.columnStats.size();
+ this.columnStats.clear();
+ for (int i = 0;i < columnSize; i++){
+ ColumnStats columnStat = new ColumnStats();
+ this.columnStats.add(columnStat);
+ }
+ return cardinality.toString();
+ }
+
public void setCardinality(String cardinality) {
if (null == cardinality)
return;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a118cae1/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
index 7058ad8..834c7f6 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
@@ -122,6 +122,10 @@ public class HiveColumnCardinalityUpdateJob extends AbstractHadoopJob {
tableExt.setCardinality(scardi);
metaMgr.saveTableExt(tableExt, prj);
} else {
+ MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+ TableExtDesc tableExt = metaMgr.getTableExt(tableName);
+ tableExt.resetCardinality();
+ metaMgr.saveTableExt(tableExt);
throw new IllegalArgumentException("No cardinality data is collected for table " + tableName);
}
}
[2/2] kylin git commit: KYLIN-2901 code review
Posted by li...@apache.org.
KYLIN-2901 code review
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0297d9ab
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0297d9ab
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0297d9ab
Branch: refs/heads/master
Commit: 0297d9abfd47fc5fb6b360d7678fc7cfac4172a4
Parents: a118cae
Author: Li Yang <li...@apache.org>
Authored: Sun Oct 8 18:40:14 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sun Oct 8 18:44:31 2017 +0800
----------------------------------------------------------------------
.../kylin/metadata/model/TableExtDesc.java | 33 ++++++++++----------
.../HiveColumnCardinalityUpdateJob.java | 6 ++--
2 files changed, 19 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/0297d9ab/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
index e4cda8f..42da32e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
@@ -61,7 +61,7 @@ public class TableExtDesc extends RootPersistentEntity {
private Map<String, String> dataSourceProps = new HashMap<>();
private String project;
-
+
public TableExtDesc() {
}
@@ -72,7 +72,7 @@ public class TableExtDesc extends RootPersistentEntity {
public static String concatRawResourcePath(String nameOnPath) {
return ResourceStore.TABLE_EXD_RESOURCE_ROOT + "/" + nameOnPath + ".json";
}
-
+
public static String concatResourcePath(String tableIdentity, String prj) {
if (prj == null)
return ResourceStore.TABLE_EXD_RESOURCE_ROOT + "/" + tableIdentity + ".json";
@@ -84,11 +84,11 @@ public class TableExtDesc extends RootPersistentEntity {
public static Pair<String, String> parseResourcePath(String path) {
return TableDesc.parseResourcePath(path);
}
-
+
public String getProject() {
return project;
}
-
+
public int getFrequency() {
return this.frequency;
}
@@ -100,7 +100,7 @@ public class TableExtDesc extends RootPersistentEntity {
public String getIdentity() {
return this.tableIdentity;
}
-
+
public String getJodID() {
return this.jodID;
}
@@ -138,18 +138,15 @@ public class TableExtDesc extends RootPersistentEntity {
}
return cardinality.toString();
}
-
- public String resetCardinality(){
- StringBuffer cardinality = new StringBuffer();
- int columnSize=this.columnStats.size();
+
+ public void resetCardinality() {
+ int columnSize = this.columnStats.size();
this.columnStats.clear();
- for (int i = 0;i < columnSize; i++){
- ColumnStats columnStat = new ColumnStats();
- this.columnStats.add(columnStat);
+ for (int i = 0; i < columnSize; i++) {
+ this.columnStats.add(new ColumnStats());
}
- return cardinality.toString();
}
-
+
public void setCardinality(String cardinality) {
if (null == cardinality)
return;
@@ -199,7 +196,7 @@ public class TableExtDesc extends RootPersistentEntity {
public void init(String project) {
this.project = project;
-
+
if (this.tableIdentity != null)
this.tableIdentity = this.tableIdentity.toUpperCase();
}
@@ -213,7 +210,8 @@ public class TableExtDesc extends RootPersistentEntity {
}
public boolean isPartitioned() {
- return this.dataSourceProps.get("partition_column") == null ? false : !this.dataSourceProps.get("partition_column").isEmpty();
+ return this.dataSourceProps.get("partition_column") == null ? false
+ : !this.dataSourceProps.get("partition_column").isEmpty();
}
@Override
@@ -230,7 +228,8 @@ public class TableExtDesc extends RootPersistentEntity {
@Override
public String toString() {
- return "TableExtDesc{" + "name='" + (null == tableIdentity ? "NULL" : tableIdentity) + '\'' + ", columns_samples=" + (null == columnStats ? "null" : Arrays.toString(columnStats.toArray()));
+ return "TableExtDesc{" + "name='" + (null == tableIdentity ? "NULL" : tableIdentity) + '\''
+ + ", columns_samples=" + (null == columnStats ? "null" : Arrays.toString(columnStats.toArray()));
}
@JsonIgnoreProperties(ignoreUnknown = true)
http://git-wip-us.apache.org/repos/asf/kylin/blob/0297d9ab/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
index 834c7f6..023ebfd 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
@@ -122,11 +122,11 @@ public class HiveColumnCardinalityUpdateJob extends AbstractHadoopJob {
tableExt.setCardinality(scardi);
metaMgr.saveTableExt(tableExt, prj);
} else {
+ // it gets here when ColumnCardinalityReducer output no record, which means empty table
MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
- TableExtDesc tableExt = metaMgr.getTableExt(tableName);
+ TableExtDesc tableExt = metaMgr.getTableExt(tableName, prj);
tableExt.resetCardinality();
- metaMgr.saveTableExt(tableExt);
- throw new IllegalArgumentException("No cardinality data is collected for table " + tableName);
+ metaMgr.saveTableExt(tableExt, prj);
}
}