You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2016/10/31 02:23:34 UTC
[02/13] kylin git commit: KYLIN-2012 adjust rules for LookupTable &
remove dead methods
KYLIN-2012 adjust rules for LookupTable & remove dead methods
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d5565f77
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d5565f77
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d5565f77
Branch: refs/heads/yang21-hbase1.x
Commit: d5565f775fa0e907a5a2c0bc33c75a2ece91ae31
Parents: f56c3c4
Author: gaodayue <ga...@meituan.com>
Authored: Thu Oct 13 16:56:05 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Oct 26 18:39:05 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/metadata/MetadataManager.java | 30 --------------------
.../apache/kylin/source/hive/SchemaChecker.java | 27 ++++++++++++++----
2 files changed, 21 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d5565f77/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index 8d77131..fd0a3c3 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@ -461,11 +461,6 @@ public class MetadataManager {
reloadSourceTableAt(TableDesc.concatResourcePath(tableIdentity));
}
- public void reloadTableCache(String tableIdentity) throws IOException {
- reloadSourceTableExt(tableIdentity);
- reloadSourceTable(tableIdentity);
- }
-
public DataModelDesc getDataModelDesc(String name) {
return dataModelDescMap.get(name);
}
@@ -622,29 +617,4 @@ public class MetadataManager {
srcTableExdMap.remove(tableIdentity);
}
- public String appendDBName(String table) {
-
- if (table.indexOf(".") > 0)
- return table;
-
- Map<String, TableDesc> map = getAllTablesMap();
-
- int count = 0;
- String result = null;
- for (TableDesc t : map.values()) {
- if (t.getName().equalsIgnoreCase(table)) {
- result = t.getIdentity();
- count++;
- }
- }
-
- if (count == 1)
- return result;
-
- if (count > 1) {
- logger.warn("There are more than 1 table named with '" + table + "' in different database; The program couldn't determine, randomly pick '" + result + "'");
- }
- return result;
- }
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d5565f77/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java b/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
index 3b03551..0c5ccd0 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
@@ -134,7 +134,14 @@ public class SchemaChecker {
}
}
- private List<String> checkAllUsedColumns(CubeInstance cube, TableDesc table, Map<String, FieldSchema> fieldsMap) {
+ /**
+ * check whether all columns used in `cube` has compatible schema in current hive schema denoted by `fieldsMap`.
+ * @param cube cube to check, must use `table` in its model
+ * @param table kylin's table metadata
+ * @param fieldsMap current hive schema of `table`
+ * @return true if all columns used in `cube` has compatible schema with `fieldsMap`, false otherwise
+ */
+ private List<String> checkAllColumnsInCube(CubeInstance cube, TableDesc table, Map<String, FieldSchema> fieldsMap) {
Set<ColumnDesc> usedColumns = Sets.newHashSet();
for (TblColRef col : cube.getAllColumns()) {
usedColumns.add(col.getColumnDesc());
@@ -152,8 +159,15 @@ public class SchemaChecker {
return violateColumns;
}
- private boolean checkAllColumns(TableDesc table, List<FieldSchema> fields) {
- if (table.getColumnCount() != fields.size()) {
+ /**
+ * check whether all columns in `table` are still in `fields` and have the same index as before.
+ *
+ * @param table kylin's table metadata
+ * @param fields current table metadata in hive
+ * @return true if only new columns are appended in hive, false otherwise
+ */
+ private boolean checkAllColumnsInTableDesc(TableDesc table, List<FieldSchema> fields) {
+ if (table.getColumnCount() > fields.size()) {
return false;
}
@@ -194,15 +208,16 @@ public class SchemaChecker {
// if user reloads a fact table used by cube, then all used columns
// must match current schema
if (factTable.getIdentity().equals(fullTableName)) {
- List<String> violateColumns = checkAllUsedColumns(cube, factTable, currentFieldsMap);
+ List<String> violateColumns = checkAllColumnsInCube(cube, factTable, currentFieldsMap);
if (!violateColumns.isEmpty()) {
issues.add(format("Column %s used in cube[%s] and model[%s], but changed in hive", violateColumns, cube.getName(), modelName));
}
}
- // if user reloads a lookup table used by cube, then nearly all changes in schema are disallowed)
+ // if user reloads a lookup table used by cube, only append column(s) are allowed, all existing columns
+ // must be the same (except compatible type changes)
for (TableDesc lookupTable : lookupTables) {
- if (lookupTable.getIdentity().equals(fullTableName) && !checkAllColumns(lookupTable, currentFields)) {
+ if (lookupTable.getIdentity().equals(fullTableName) && !checkAllColumnsInTableDesc(lookupTable, currentFields)) {
issues.add(format("Table '%s' is used as Lookup Table in cube[%s] and model[%s], but changed in hive", lookupTable.getIdentity(), cube.getName(), modelName));
}
}