You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jd...@apache.org on 2016/05/01 03:30:57 UTC
[03/11] hive git commit: HIVE-13572 : Redundant setting full file
status in Hive::copyFiles (Rui Li via Ashutosh Chauhan)
HIVE-13572 : Redundant setting full file status in Hive::copyFiles (Rui Li via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/076f3655
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/076f3655
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/076f3655
Branch: refs/heads/llap
Commit: 076f3655b846f3214bc304e405d321222a3819ab
Parents: 4377c7f
Author: Rui Li <ru...@intel.com>
Authored: Tue Apr 26 18:14:00 2016 -0800
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Thu Apr 28 19:22:13 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/metadata/Hive.java | 39 +++++++++++---------
.../apache/hadoop/hive/shims/ShimLoader.java | 10 +++--
2 files changed, 28 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/076f3655/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index ab165f1..4d9c3d2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -2641,26 +2641,29 @@ private void constructOneLBLocationMap(FileStatus fSta,
files = new FileStatus[] {src};
}
- for (FileStatus srcFile : files) {
-
- final Path srcP = srcFile.getPath();
- final boolean needToCopy = needToCopy(srcP, destf, srcFs, destFs);
- // Strip off the file type, if any so we don't make:
- // 000000_0.gz -> 000000_0.gz_copy_1
- final String name;
- final String filetype;
- String itemName = srcP.getName();
- int index = itemName.lastIndexOf('.');
- if (index >= 0) {
- filetype = itemName.substring(index);
- name = itemName.substring(0, index);
- } else {
- name = itemName;
- filetype = "";
- }
+ final SessionState parentSession = SessionState.get();
+ for (final FileStatus srcFile : files) {
+
futures.add(pool.submit(new Callable<ObjectPair<Path, Path>>() {
@Override
public ObjectPair<Path, Path> call() throws Exception {
+ SessionState.setCurrentSessionState(parentSession);
+ final Path srcP = srcFile.getPath();
+ final boolean needToCopy = needToCopy(srcP, destf, srcFs, destFs);
+ // Strip off the file type, if any so we don't make:
+ // 000000_0.gz -> 000000_0.gz_copy_1
+ final String name;
+ final String filetype;
+ String itemName = srcP.getName();
+ int index = itemName.lastIndexOf('.');
+ if (index >= 0) {
+ filetype = itemName.substring(index);
+ name = itemName.substring(0, index);
+ } else {
+ name = itemName;
+ filetype = "";
+ }
+
Path destPath = new Path(destf, srcP.getName());
if (!needToCopy && !isSrcLocal) {
for (int counter = 1; !destFs.rename(srcP,destPath); counter++) {
@@ -2671,7 +2674,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
}
if (inheritPerms) {
- ShimLoader.getHadoopShims().setFullFileStatus(conf, fullDestStatus, destFs, destf);
+ ShimLoader.getHadoopShims().setFullFileStatus(conf, fullDestStatus, destFs, destPath);
}
if (null != newFiles) {
newFiles.add(destPath);
http://git-wip-us.apache.org/repos/asf/hive/blob/076f3655/shims/common/src/main/java/org/apache/hadoop/hive/shims/ShimLoader.java
----------------------------------------------------------------------
diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/shims/ShimLoader.java b/shims/common/src/main/java/org/apache/hadoop/hive/shims/ShimLoader.java
index 0fe3169..28d3e48 100644
--- a/shims/common/src/main/java/org/apache/hadoop/hive/shims/ShimLoader.java
+++ b/shims/common/src/main/java/org/apache/hadoop/hive/shims/ShimLoader.java
@@ -31,7 +31,7 @@ import java.util.Map;
public abstract class ShimLoader {
public static String HADOOP23VERSIONNAME = "0.23";
- private static HadoopShims hadoopShims;
+ private static volatile HadoopShims hadoopShims;
private static JettyShims jettyShims;
private static AppenderSkeleton eventCounter;
private static HadoopThriftAuthBridge hadoopThriftAuthBridge;
@@ -88,9 +88,13 @@ public abstract class ShimLoader {
* Factory method to get an instance of HadoopShims based on the
* version of Hadoop on the classpath.
*/
- public static synchronized HadoopShims getHadoopShims() {
+ public static HadoopShims getHadoopShims() {
if (hadoopShims == null) {
- hadoopShims = loadShims(HADOOP_SHIM_CLASSES, HadoopShims.class);
+ synchronized (ShimLoader.class) {
+ if (hadoopShims == null) {
+ hadoopShims = loadShims(HADOOP_SHIM_CLASSES, HadoopShims.class);
+ }
+ }
}
return hadoopShims;
}