You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ma...@apache.org on 2022/04/01 07:18:47 UTC
[hive] branch master updated: HIVE-26098 : Duplicate path/Jar in hive.aux.jars.path or hive.reloadable.aux.jars.path causing IllegalArgumentException (Mahesh Kumar Behera, reviewed by Ayush Saxena)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new ef36922 HIVE-26098 : Duplicate path/Jar in hive.aux.jars.path or hive.reloadable.aux.jars.path causing IllegalArgumentException (Mahesh Kumar Behera, reviewed by Ayush Saxena)
ef36922 is described below
commit ef369227ea2eb9bf8b0b54147e7231364d79810a
Author: mahesh kumar behera <ma...@apache.org>
AuthorDate: Fri Apr 1 12:48:34 2022 +0530
HIVE-26098 : Duplicate path/Jar in hive.aux.jars.path or hive.reloadable.aux.jars.path causing IllegalArgumentException (Mahesh Kumar Behera, reviewed by Ayush Saxena)
---
.../apache/hadoop/hive/ql/exec/tez/DagUtils.java | 22 +++++++++---------
.../test/queries/clientpositive/whroot_external1.q | 16 +++++++++++++
.../clientpositive/llap/whroot_external1.q.out | 27 ++++++++++++++++++++++
3 files changed, 54 insertions(+), 11 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
index 73ba430..7622bcc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
@@ -1406,9 +1406,14 @@ public class DagUtils {
public LocalResource localizeResource(
Path src, Path dest, LocalResourceType type, Configuration conf) throws IOException {
FileSystem destFS = dest.getFileSystem(conf);
- // We call copyFromLocal below, so we basically assume src is a local file.
- FileSystem srcFs = FileSystem.getLocal(conf);
- if (src != null && !checkPreExisting(srcFs, src, dest, conf)) {
+ FileSystem srcFs;
+ if (src.toUri().getScheme() != null) {
+ srcFs = src.getFileSystem(conf);
+ } else {
+ srcFs = FileSystem.getLocal(conf);
+ }
+
+ if (!checkPreExisting(srcFs, src, dest, conf)) {
// copy the src to the destination and create local resource.
// do not overwrite.
String srcStr = src.toString();
@@ -1424,12 +1429,8 @@ public class DagUtils {
return createLocalResource(destFS, dest, type, LocalResourceVisibility.PRIVATE);
}
try {
- if (src.toUri().getScheme()!=null) {
- FileUtil.copy(src.getFileSystem(conf), src, destFS, dest, false, false, conf);
- }
- else {
- destFS.copyFromLocalFile(false, false, src, dest);
- }
+ // FileUtil.copy takes care of copy from local filesystem internally.
+ FileUtil.copy(srcFs, src, destFS, dest, false, false, conf);
synchronized (notifier) {
notifier.notifyAll(); // Notify if we have successfully copied the file.
}
@@ -1458,8 +1459,7 @@ public class DagUtils {
}
}
}
- return createLocalResource(destFS, dest, type,
- LocalResourceVisibility.PRIVATE);
+ return createLocalResource(destFS, dest, type, LocalResourceVisibility.PRIVATE);
}
public boolean checkOrWaitForTheFile(FileSystem srcFs, Path src, Path dest, Configuration conf,
diff --git a/ql/src/test/queries/clientpositive/whroot_external1.q b/ql/src/test/queries/clientpositive/whroot_external1.q
index c3724a4..88c253c 100644
--- a/ql/src/test/queries/clientpositive/whroot_external1.q
+++ b/ql/src/test/queries/clientpositive/whroot_external1.q
@@ -112,3 +112,19 @@ dfs -copyFromLocal ../../data/files/test_dec_space.csv hdfs:///tmp/test_dec_spac
create external table test_dec_space (id int, value decimal) ROW FORMAT DELIMITED
FIELDS TERMINATED BY ',' location 'hdfs:///tmp/test_dec_space';
select * from test_dec_space;
+
+create table tbl (fld int);
+dfs -mkdir -p hdfs:///tmp/test_load_aux_jar;
+dfs -copyFromLocal ${system:hive.root}/data/files/identity_udf.jar hdfs:///tmp/test_load_aux_jar/;
+
+-- both hive.aux.jars.path and hive.reloadable.aux.jars.path pointing to the same jar.
+SET hive.aux.jars.path=hdfs:///tmp/test_load_aux_jar/identity_udf.jar;
+SET hive.reloadable.aux.jars.path=hdfs:///tmp/test_load_aux_jar/;
+
+-- reload will load the identity_udf.jar from tmp/test_load_aux_jar
+RELOAD;
+
+insert into tbl values(1);
+select * from tbl;
+
+dfs -rmr -f hdfs:///tmp/test_load_aux_jar/
\ No newline at end of file
diff --git a/ql/src/test/results/clientpositive/llap/whroot_external1.q.out b/ql/src/test/results/clientpositive/llap/whroot_external1.q.out
index 94fd2ae..f9716e6 100644
--- a/ql/src/test/results/clientpositive/llap/whroot_external1.q.out
+++ b/ql/src/test/results/clientpositive/llap/whroot_external1.q.out
@@ -614,3 +614,30 @@ POSTHOOK: Input: default@test_dec_space
POSTHOOK: Output: hdfs://### HDFS PATH ###
1 0
2 1
+PREHOOK: query: create table tbl (fld int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@tbl
+POSTHOOK: query: create table tbl (fld int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@tbl
+PREHOOK: query: insert into tbl values(1)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@tbl
+POSTHOOK: query: insert into tbl values(1)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@tbl
+POSTHOOK: Lineage: tbl.fld SCRIPT []
+PREHOOK: query: select * from tbl
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from tbl
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1
+#### A masked pattern was here ####