You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/03/14 01:02:34 UTC
[22/47] incubator-kylin git commit: KYLIN-627 Hive tables' partition
column was not sync into Kylin
KYLIN-627 Hive tables' partition column was not sync into Kylin
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/afff4e76
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/afff4e76
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/afff4e76
Branch: refs/heads/master
Commit: afff4e76a8e7a11549efeac57f845694d9f558a2
Parents: cfbd6ae
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Mon Mar 9 17:09:16 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Mon Mar 9 17:09:16 2015 +0800
----------------------------------------------------------------------
.../kylin/metadata/tool/HiveSourceTableLoader.java | 13 +++++++++----
1 file changed, 9 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/afff4e76/metadata/src/main/java/org/apache/kylin/metadata/tool/HiveSourceTableLoader.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/tool/HiveSourceTableLoader.java b/metadata/src/main/java/org/apache/kylin/metadata/tool/HiveSourceTableLoader.java
index 00891d9..5297188 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/tool/HiveSourceTableLoader.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/tool/HiveSourceTableLoader.java
@@ -81,15 +81,21 @@ public class HiveSourceTableLoader {
for (String tableName : tables) {
Table table = null;
HiveClient hiveClient = new HiveClient();
+ List<FieldSchema> partitionFields = null;
List<FieldSchema> fields = null;
try {
table = hiveClient.getHiveTable(database, tableName);
+ partitionFields = table.getPartitionKeys();
fields = hiveClient.getHiveTableFields(database, tableName);
} catch (Exception e) {
e.printStackTrace();
throw new IOException(e);
}
+ if (fields != null && partitionFields != null && partitionFields.size() > 0) {
+ fields.addAll(partitionFields);
+ }
+
long tableSize = hiveClient.getFileSizeForTable(table);
long tableFileNum = hiveClient.getFileNumberForTable(table);
TableDesc tableDesc = metaMgr.getTableDesc(database + "." + tableName);
@@ -113,12 +119,11 @@ public class HiveSourceTableLoader {
}
tableDesc.setColumns(columns.toArray(new ColumnDesc[columnNumber]));
- List<FieldSchema> partitionCols = table.getPartitionKeys();
StringBuffer partitionColumnString = new StringBuffer();
- for (int i = 0, n = partitionCols.size(); i < n; i++) {
+ for (int i = 0, n = partitionFields.size(); i < n; i++) {
if (i > 0)
partitionColumnString.append(", ");
- partitionColumnString.append(partitionCols.get(i).getName().toUpperCase());
+ partitionColumnString.append(partitionFields.get(i).getName().toUpperCase());
}
Map<String, String> map = metaMgr.getTableDescExd(tableDesc.getIdentity());
@@ -135,7 +140,7 @@ public class HiveSourceTableLoader {
map.put(MetadataConstants.TABLE_EXD_PC, partitionColumnString.toString());
map.put(MetadataConstants.TABLE_EXD_TFS, String.valueOf(tableSize));
map.put(MetadataConstants.TABLE_EXD_TNF, String.valueOf(tableFileNum));
- map.put(MetadataConstants.TABLE_EXD_PARTITIONED, Boolean.valueOf(partitionCols != null && partitionCols.size()>0).toString());
+ map.put(MetadataConstants.TABLE_EXD_PARTITIONED, Boolean.valueOf(partitionFields != null && partitionFields.size() > 0).toString());
metaMgr.saveSourceTable(tableDesc);
metaMgr.saveTableExd(tableDesc.getIdentity(), map);