You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/07/01 08:23:27 UTC
incubator-kylin git commit: KYLIN-824 Cube Build fails if lookup
table doesn't have any files under HDFS location
Repository: incubator-kylin
Updated Branches:
refs/heads/0.7-staging e7fcc20d8 -> 09751393f
KYLIN-824 Cube Build fails if lookup table doesn't have any files under HDFS location
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/09751393
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/09751393
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/09751393
Branch: refs/heads/0.7-staging
Commit: 09751393f9abe6eec8e087b45d6b422c12b5f105
Parents: e7fcc20
Author: shaofengshi <sh...@apache.org>
Authored: Tue Jun 30 10:22:11 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Wed Jul 1 14:22:53 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/common/util/HiveClient.java | 21 +++++++++++--
.../org/apache/kylin/dict/lookup/FileTable.java | 18 ++++++++++--
.../org/apache/kylin/dict/lookup/HiveTable.java | 31 +++++++++++++++-----
3 files changed, 57 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/09751393/common/src/main/java/org/apache/kylin/common/util/HiveClient.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/HiveClient.java b/common/src/main/java/org/apache/kylin/common/util/HiveClient.java
index cfeea31..86a403d 100644
--- a/common/src/main/java/org/apache/kylin/common/util/HiveClient.java
+++ b/common/src/main/java/org/apache/kylin/common/util/HiveClient.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.ql.CommandNeedRetryException;
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
@@ -44,6 +45,7 @@ public class HiveClient {
protected HiveConf hiveConf = null;
protected Driver driver = null;
protected HiveMetaStoreClient metaStoreClient = null;
+ protected String type;
public HiveClient() {
hiveConf = new HiveConf(HiveClient.class);
@@ -86,8 +88,8 @@ public class HiveClient {
/**
*
* @param hql
- * @throws CommandNeedRetryException
- * @throws IOException
+ * @throws org.apache.hadoop.hive.ql.CommandNeedRetryException
+ * @throws java.io.IOException
*/
public void executeHQL(String hql) throws CommandNeedRetryException, IOException {
CommandProcessorResponse response = getDriver().run(hql);
@@ -110,6 +112,7 @@ public class HiveClient {
return metaStoreClient;
}
+
public Table getHiveTable(String database, String tableName) throws Exception {
return getMetaStoreClient().getTable(database, tableName);
}
@@ -155,4 +158,18 @@ public class HiveClient {
return result;
}
+ /**
+ *
+ * @param database
+ * @param tableName
+ * @throws Exception
+ */
+
+ public boolean isNativeTable(String database, String tableName) throws Exception{
+
+ return !MetaStoreUtils.isNonNativeTable(getMetaStoreClient().getTable(database, tableName));
+
+
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/09751393/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTable.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTable.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTable.java
index e6e3de9..a659ca3 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTable.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTable.java
@@ -35,15 +35,24 @@ public class FileTable implements ReadableTable {
String path;
String delim;
int nColumns;
+ boolean nativeTable;
public FileTable(String path, int nColumns) {
- this(path, DELIM_AUTO, nColumns);
+ this(path, DELIM_AUTO, nColumns, true);
}
- public FileTable(String path, String delim, int nColumns) {
+ public FileTable(String path, String delim, int nColumns, boolean nativeTable) {
this.path = path;
this.delim = delim;
this.nColumns = nColumns;
+ this.nativeTable = nativeTable;
+ }
+
+ public FileTable(String path, int nColumns, boolean nativeTable) {
+ this.path = path;
+ this.delim = DELIM_AUTO;
+ this.nColumns = nColumns;
+ this.nativeTable = nativeTable;
}
@Override
@@ -60,7 +69,10 @@ public class FileTable implements ReadableTable {
public TableSignature getSignature() throws IOException {
FileSystem fs = HadoopUtil.getFileSystem(path);
FileStatus status = fs.getFileStatus(new Path(path));
- return new TableSignature(path, status.getLen(), status.getModificationTime());
+ if (nativeTable) {
+ return new TableSignature(path, status.getLen(), status.getModificationTime());
+ }
+ return new TableSignature(path, status.getLen(), System.currentTimeMillis());
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/09751393/dictionary/src/main/java/org/apache/kylin/dict/lookup/HiveTable.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/HiveTable.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/HiveTable.java
index ae224ac..aa22814 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/HiveTable.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/lookup/HiveTable.java
@@ -48,6 +48,8 @@ public class HiveTable implements ReadableTable {
private int nColumns;
private String hdfsLocation;
private FileTable fileTable;
+ private HiveClient hiveClient;
+ private boolean nativeTable;
public HiveTable(MetadataManager metaMgr, String table) {
TableDesc tableDesc = metaMgr.getTableDesc(table);
@@ -72,20 +74,26 @@ public class HiveTable implements ReadableTable {
}
private FileTable getFileTable() throws IOException {
- if (fileTable == null) {
- fileTable = new FileTable(getHDFSLocation(true), nColumns);
+ try {
+ if (fileTable == null) {
+ nativeTable = getHiveClient().isNativeTable(database, hiveTable);
+ fileTable = new FileTable(getHDFSLocation(), nColumns, nativeTable);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw new IOException(e);
}
return fileTable;
}
- public String getHDFSLocation(boolean needFilePath) throws IOException {
+ public String getHDFSLocation() throws IOException {
if (hdfsLocation == null) {
- hdfsLocation = computeHDFSLocation(needFilePath);
+ hdfsLocation = computeHDFSLocation();
}
return hdfsLocation;
}
- private String computeHDFSLocation(boolean needFilePath) throws IOException {
+ private String computeHDFSLocation() throws IOException {
String override = KylinConfig.getInstanceFromEnv().getOverrideHiveTableLocation(hiveTable);
if (override != null) {
@@ -95,14 +103,13 @@ public class HiveTable implements ReadableTable {
String hdfsDir = null;
try {
- HiveClient hiveClient = new HiveClient();
- hdfsDir = hiveClient.getHiveTableLocation(database, hiveTable);
+ hdfsDir = getHiveClient().getHiveTableLocation(database, hiveTable);
} catch (Exception e) {
e.printStackTrace();
throw new IOException(e);
}
- if (needFilePath) {
+ if (nativeTable) {
FileSystem fs = HadoopUtil.getFileSystem(hdfsDir);
FileStatus file = findOnlyFile(hdfsDir, fs);
return file.getPath().toString();
@@ -129,4 +136,12 @@ public class HiveTable implements ReadableTable {
return "hive: database=[" + database + "], table=[" + hiveTable + "]";
}
+ public HiveClient getHiveClient() {
+
+ if (hiveClient == null) {
+ hiveClient = new HiveClient();
+ }
+ return hiveClient;
+ }
+
}