You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ab...@apache.org on 2020/10/28 21:12:56 UTC
[hive] branch master updated (f7e9d9b -> ebb13d0)
This is an automated email from the ASF dual-hosted git repository.
abstractdog pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git.
from f7e9d9b HIVE-24270: Move scratchdir cleanup to background
new 86f73ab HIVE-24108: AddToClassPathAction should use TezClassLoader (László Bodor reviewed by Harish JP, Ashutosh Chauhan)
new 416edbb HIVE-23190: LLAP: modify IndexCache to pass filesystem object to TezSpillRecord (László Bodor reviewed by Rajesh Balamohan)
new ebb13d0 HIVE-23930: Upgrade to tez 0.10.0
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
itests/src/test/resources/testconfiguration.properties | 1 +
.../hadoop/hive/llap/daemon/impl/FunctionLocalizer.java | 3 ++-
.../apache/hadoop/hive/llap/shufflehandler/IndexCache.java | 13 ++++++++++++-
pom.xml | 2 +-
.../clientpositive/{llap => tez}/mapjoin_addjar.q.out | 4 ++--
5 files changed, 18 insertions(+), 5 deletions(-)
rename ql/src/test/results/clientpositive/{llap => tez}/mapjoin_addjar.q.out (99%)
[hive] 02/03: HIVE-23190: LLAP: modify IndexCache to pass filesystem object to TezSpillRecord (László Bodor reviewed by Rajesh Balamohan)
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abstractdog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
commit 416edbb4a1a6148afe87a628589a6d2a6da84c91
Author: László Bodor <bo...@gmail.com>
AuthorDate: Wed Oct 28 22:04:22 2020 +0100
HIVE-23190: LLAP: modify IndexCache to pass filesystem object to TezSpillRecord (László Bodor reviewed by Rajesh Balamohan)
Signed-off-by: Laszlo Bodor <bo...@gmail.com>
---
.../apache/hadoop/hive/llap/shufflehandler/IndexCache.java | 13 ++++++++++++-
1 file changed, 12 insertions(+), 1 deletion(-)
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/IndexCache.java b/llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/IndexCache.java
index fb1bcfe..63c91da 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/IndexCache.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/IndexCache.java
@@ -25,6 +25,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.tez.runtime.library.common.Constants;
import org.apache.tez.runtime.library.common.sort.impl.TezIndexRecord;
@@ -42,12 +43,22 @@ class IndexCache {
private final LinkedBlockingQueue<String> queue =
new LinkedBlockingQueue<String>();
+ private FileSystem fs;
public static final String INDEX_CACHE_MB = "llap.shuffle.indexcache.mb";
public IndexCache(Configuration conf) {
this.conf = conf;
totalMemoryAllowed = conf.getInt(INDEX_CACHE_MB, 10) * 1024 * 1024;
LOG.info("IndexCache created with max memory = " + totalMemoryAllowed);
+ initLocalFs();
+ }
+
+ private void initLocalFs() {
+ try {
+ this.fs = FileSystem.getLocal(conf).getRaw();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
}
/**
@@ -119,7 +130,7 @@ class IndexCache {
LOG.debug("IndexCache MISS: MapId " + mapId + " not found") ;
TezSpillRecord tmp = null;
try {
- tmp = new TezSpillRecord(indexFileName, conf, expectedIndexOwner);
+ tmp = new TezSpillRecord(indexFileName, fs, expectedIndexOwner);
} catch (Throwable e) {
tmp = new TezSpillRecord(0);
cache.remove(mapId);
[hive] 01/03: HIVE-24108: AddToClassPathAction should use TezClassLoader (László Bodor reviewed by Harish JP, Ashutosh Chauhan)
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abstractdog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
commit 86f73ab199965f538c2dae7fbd679882b423087b
Author: László Bodor <bo...@gmail.com>
AuthorDate: Wed Oct 28 22:03:42 2020 +0100
HIVE-24108: AddToClassPathAction should use TezClassLoader (László Bodor reviewed by Harish JP, Ashutosh Chauhan)
Signed-off-by: Laszlo Bodor <bo...@gmail.com>
---
itests/src/test/resources/testconfiguration.properties | 1 +
.../org/apache/hadoop/hive/llap/daemon/impl/FunctionLocalizer.java | 3 ++-
ql/src/test/results/clientpositive/{llap => tez}/mapjoin_addjar.q.out | 4 ++--
3 files changed, 5 insertions(+), 3 deletions(-)
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 217a819..8372843 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -20,6 +20,7 @@ minitez.query.files=\
explainanalyze_4.q,\
explainanalyze_5.q,\
explainuser_3.q,\
+ mapjoin_addjar.q,\
multi_count_distinct.q,\
orc_merge12.q,\
orc_vectorization_ppd.q,\
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/FunctionLocalizer.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/FunctionLocalizer.java
index 315c5be..e50c11a 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/FunctionLocalizer.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/FunctionLocalizer.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.session.SessionState.ResourceType;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
import org.apache.hadoop.hive.ql.util.ResourceDownloader;
+import org.apache.tez.common.TezClassLoader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -73,7 +74,7 @@ public class FunctionLocalizer implements GenericUDFBridge.UdfWhitelistChecker {
this.conf = conf;
this.localDir = new File(localDir, DIR_NAME);
AddToClassPathAction addAction = new AddToClassPathAction(
- Thread.currentThread().getContextClassLoader(), Collections.emptyList(), true);
+ TezClassLoader.getInstance(), Collections.emptyList(), true);
this.executorClassloader = AccessController.doPrivileged(addAction);
this.workThread = new Thread(new Runnable() {
@Override
diff --git a/ql/src/test/results/clientpositive/llap/mapjoin_addjar.q.out b/ql/src/test/results/clientpositive/tez/mapjoin_addjar.q.out
similarity index 99%
rename from ql/src/test/results/clientpositive/llap/mapjoin_addjar.q.out
rename to ql/src/test/results/clientpositive/tez/mapjoin_addjar.q.out
index 41f5adb..a8b94f5 100644
--- a/ql/src/test/results/clientpositive/llap/mapjoin_addjar.q.out
+++ b/ql/src/test/results/clientpositive/tez/mapjoin_addjar.q.out
@@ -17,11 +17,11 @@ POSTHOOK: Output: default@t1_n66
PREHOOK: query: select * from t1_n66 l join t1_n66 r on l.key =r.key
PREHOOK: type: QUERY
PREHOOK: Input: default@t1_n66
-#### A masked pattern was here ####
+PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: select * from t1_n66 l join t1_n66 r on l.key =r.key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@t1_n66
-#### A masked pattern was here ####
+POSTHOOK: Output: hdfs://### HDFS PATH ###
238 val_238 238 val_238
238 val_238 238 val_238
86 val_86 86 val_86
[hive] 03/03: HIVE-23930: Upgrade to tez 0.10.0
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
abstractdog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
commit ebb13d0f12830f46573fb9b108d0f05bcf0a00b7
Author: Laszlo Bodor <bo...@gmail.com>
AuthorDate: Wed Oct 28 22:05:53 2020 +0100
HIVE-23930: Upgrade to tez 0.10.0
---
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pom.xml b/pom.xml
index dbf13c0..580b8a0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -194,7 +194,7 @@
<slf4j.version>1.7.30</slf4j.version>
<ST4.version>4.0.4</ST4.version>
<storage-api.version>2.7.0-SNAPSHOT</storage-api.version>
- <tez.version>0.9.1</tez.version>
+ <tez.version>0.10.0</tez.version>
<super-csv.version>2.2.0</super-csv.version>
<spark.version>2.4.5</spark.version>
<scala.binary.version>2.12</scala.binary.version>