[incubator-druid] branch master updated: make hdfs index map reduce task add jar more reasonable (#7294)

[gi...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

gian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git


The following commit(s) were added to refs/heads/master by this push:
     new 6789438  make hdfs index map reduce task add jar more reasonable (#7294)
6789438 is described below

commit 6789438a496d103b861bda7d8e6d6299715b7fb1
Author: Faxian Zhao <23...@qq.com>
AuthorDate: Mon Apr 15 01:26:59 2019 +0800

    make hdfs index map reduce task add jar more reasonable (#7294)
---
 .../main/java/org/apache/druid/indexer/JobHelper.java | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/indexing-hadoop/src/main/java/org/apache/druid/indexer/JobHelper.java b/indexing-hadoop/src/main/java/org/apache/druid/indexer/JobHelper.java
index 19303ff..44d68b1 100644
--- a/indexing-hadoop/src/main/java/org/apache/druid/indexer/JobHelper.java
+++ b/indexing-hadoop/src/main/java/org/apache/druid/indexer/JobHelper.java
@@ -52,6 +52,7 @@ import org.apache.hadoop.util.Progressable;
 import java.io.BufferedOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
+import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
@@ -209,7 +210,7 @@ public class JobHelper
 
     // Non-snapshot jar files are uploaded to the shared classpath.
     final Path hdfsPath = new Path(distributedClassPath, jarFile.getName());
-    if (!fs.exists(hdfsPath)) {
+    if (shouldUploadOrReplace(jarFile, hdfsPath, fs)) {
       // Muliple jobs can try to upload the jar here,
       // to avoid them from overwriting files, first upload to intermediateClassPath and then rename to the distributedClasspath.
       final Path intermediateHdfsPath = new Path(intermediateClassPath, jarFile.getName());
@@ -256,6 +257,22 @@ public class JobHelper
     job.addFileToClassPath(hdfsPath);
   }
 
+  static boolean shouldUploadOrReplace(
+      File jarFile,
+      Path hdfsPath,
+      FileSystem fs
+  )
+      throws IOException
+  {
+    try {
+      FileStatus status = fs.getFileStatus(hdfsPath);
+      return status == null || status.getLen() != jarFile.length();
+    }
+    catch (FileNotFoundException e) {
+      return true;
+    }
+  }
+
   static void addSnapshotJarToClassPath(
       File jarFile,
       Path intermediateClassPath,


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org