You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2016/12/19 02:15:28 UTC
[5/8] kylin git commit: KYLIN-2291 Collect hive table property
skip-header-line-count
KYLIN-2291 Collect hive table property skip-header-line-count
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bcf2d0ef
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bcf2d0ef
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bcf2d0ef
Branch: refs/heads/yang22-cdh5.7
Commit: bcf2d0ef25013306798c54bb4e485bdfbe42f35c
Parents: 82f6c55
Author: Cheng Wang <ch...@kyligence.io>
Authored: Fri Dec 16 17:23:15 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Dec 16 17:52:26 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/source/hive/BeelineHiveClient.java | 3 +++
.../java/org/apache/kylin/source/hive/CLIHiveClient.java | 2 +-
.../apache/kylin/source/hive/HiveSourceTableLoader.java | 1 +
.../java/org/apache/kylin/source/hive/HiveTableMeta.java | 4 +++-
.../apache/kylin/source/hive/HiveTableMetaBuilder.java | 11 ++++++++++-
5 files changed, 18 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/bcf2d0ef/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
index 47b551b..468ccb1 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
@@ -193,6 +193,9 @@ public class BeelineHiveClient implements IHiveClient {
if ("numFiles".equals(resultSet.getString(2).trim())) {
builder.setFileNum(Long.parseLong(resultSet.getString(3).trim()));
}
+ if ("skip.header.line.count".equals(resultSet.getString(2).trim())) {
+ builder.setSkipHeaderLineCount(resultSet.getString(3).trim());
+ }
}
}
if ("InputFormat:".equals(resultSet.getString(1).trim())) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/bcf2d0ef/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
index 5a17f1f..e8a93bd 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
@@ -42,7 +42,6 @@ import com.google.common.collect.Lists;
*
*/
public class CLIHiveClient implements IHiveClient {
-
protected HiveConf hiveConf = null;
protected Driver driver = null;
protected HiveMetaStoreClient metaStoreClient = null;
@@ -109,6 +108,7 @@ public class CLIHiveClient implements IHiveClient {
builder.setOwner(table.getOwner());
builder.setLastAccessTime(table.getLastAccessTime());
builder.setTableType(table.getTableType());
+ builder.setSkipHeaderLineCount(table.getParameters().get("skip.header.line.count"));
return builder.createHiveTableMeta();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/bcf2d0ef/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
index 57292dc..77e1084 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
@@ -140,6 +140,7 @@ public class HiveSourceTableLoader {
tableExtDesc.addDataSourceProp("total_file_number", String.valueOf(hiveTableMeta.fileNum));
tableExtDesc.addDataSourceProp("hive_inputFormat", hiveTableMeta.sdInputFormat);
tableExtDesc.addDataSourceProp("hive_outputFormat", hiveTableMeta.sdOutputFormat);
+ tableExtDesc.addDataSourceProp("skip_header_line_count", String.valueOf(hiveTableMeta.skipHeaderLineCount));
metaMgr.saveTableExt(tableExtDesc);
metaMgr.saveSourceTable(tableDesc);
http://git-wip-us.apache.org/repos/asf/kylin/blob/bcf2d0ef/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java
index 784a0bb..fa9eb29 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java
@@ -44,6 +44,7 @@ class HiveTableMeta {
String sdOutputFormat;
String owner;
String tableType;
+ int skipHeaderLineCount;
int lastAccessTime;
long fileSize;
long fileNum;
@@ -51,7 +52,7 @@ class HiveTableMeta {
List<HiveTableColumnMeta> allColumns;
List<HiveTableColumnMeta> partitionColumns;
- public HiveTableMeta(String tableName, String sdLocation, String sdInputFormat, String sdOutputFormat, String owner, String tableType, int lastAccessTime, long fileSize, long fileNum, boolean isNative, List<HiveTableColumnMeta> allColumns, List<HiveTableColumnMeta> partitionColumns) {
+ public HiveTableMeta(String tableName, String sdLocation, String sdInputFormat, String sdOutputFormat, String owner, String tableType, int lastAccessTime, long fileSize, long fileNum, int skipHeaderLineCount, boolean isNative, List<HiveTableColumnMeta> allColumns, List<HiveTableColumnMeta> partitionColumns) {
this.tableName = tableName;
this.sdLocation = sdLocation;
this.sdInputFormat = sdInputFormat;
@@ -64,6 +65,7 @@ class HiveTableMeta {
this.isNative = isNative;
this.allColumns = allColumns;
this.partitionColumns = partitionColumns;
+ this.skipHeaderLineCount = skipHeaderLineCount;
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/bcf2d0ef/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMetaBuilder.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMetaBuilder.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMetaBuilder.java
index 7a3e5d6..073ded5 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMetaBuilder.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMetaBuilder.java
@@ -32,6 +32,7 @@ public class HiveTableMetaBuilder {
private int lastAccessTime;
private long fileSize;
private long fileNum;
+ private int skipHeaderLineCount;
private boolean isNative = true;
private List<HiveTableMeta.HiveTableColumnMeta> allColumns = Lists.newArrayList();
private List<HiveTableMeta.HiveTableColumnMeta> partitionColumns = Lists.newArrayList();
@@ -81,6 +82,14 @@ public class HiveTableMetaBuilder {
return this;
}
+ public HiveTableMetaBuilder setSkipHeaderLineCount(String skipHeaderLineCount) {
+ if (null == skipHeaderLineCount)
+ this.skipHeaderLineCount = 0;
+ else
+ this.skipHeaderLineCount = Integer.parseInt(skipHeaderLineCount);
+ return this;
+ }
+
public HiveTableMetaBuilder setIsNative(boolean isNative) {
this.isNative = isNative;
return this;
@@ -97,6 +106,6 @@ public class HiveTableMetaBuilder {
}
public HiveTableMeta createHiveTableMeta() {
- return new HiveTableMeta(tableName, sdLocation, sdInputFormat, sdOutputFormat, owner, tableType, lastAccessTime, fileSize, fileNum, isNative, allColumns, partitionColumns);
+ return new HiveTableMeta(tableName, sdLocation, sdInputFormat, sdOutputFormat, owner, tableType, lastAccessTime, fileSize, fileNum, skipHeaderLineCount, isNative, allColumns, partitionColumns);
}
}
\ No newline at end of file