You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ni...@apache.org on 2019/11/26 10:39:44 UTC
[kylin] branch master updated: KYLIN-4190 hiveproducer write()
function throw exception because hive mertics table location path prefix is
different with defaut fs when hdfs uses router-based federation
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new 1868947 KYLIN-4190 hiveproducer write() function throw exception because hive mertics table location path prefix is different with defaut fs when hdfs uses router-based federation
1868947 is described below
commit 186894778d30b60bee6da93d69a8cd7c76ccef4f
Author: chuxiao <cr...@163.com>
AuthorDate: Fri Oct 11 15:07:56 2019 +0800
KYLIN-4190 hiveproducer write() function throw exception because hive mertics table location path prefix is different with defaut fs when hdfs uses router-based federation
---
.../java/org/apache/kylin/metrics/lib/impl/hive/HiveProducer.java | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/metrics-reporter-hive/src/main/java/org/apache/kylin/metrics/lib/impl/hive/HiveProducer.java b/metrics-reporter-hive/src/main/java/org/apache/kylin/metrics/lib/impl/hive/HiveProducer.java
index 74e389e..8b67e8a 100644
--- a/metrics-reporter-hive/src/main/java/org/apache/kylin/metrics/lib/impl/hive/HiveProducer.java
+++ b/metrics-reporter-hive/src/main/java/org/apache/kylin/metrics/lib/impl/hive/HiveProducer.java
@@ -57,7 +57,7 @@ public class HiveProducer {
private static final Logger logger = LoggerFactory.getLogger(HiveProducer.class);
private static final int CACHE_MAX_SIZE = 10;
private final HiveConf hiveConf;
- private final FileSystem fs;
+ private FileSystem fs;
private final LoadingCache<Pair<String, String>, Pair<String, List<FieldSchema>>> tableFieldSchemaCache;
private final String contentFilePrefix;
private String metricType;
@@ -149,6 +149,11 @@ public class HiveProducer {
sb.append(e.getValue());
}
Path partitionPath = new Path(sb.toString());
+ //for hdfs router-based federation, authority is different with hive table location path and defaultFs
+ if (partitionPath.toUri().getScheme() != null && !partitionPath.toUri().toString().startsWith(fs.getUri().toString())) {
+ fs.close();
+ fs = partitionPath.getFileSystem(hiveConf);
+ }
// Step 2: create partition for hive table if not exists
if (!fs.exists(partitionPath)) {