[GitHub] oleewere closed pull request #30: AMBARI-24833. Let logfeeder.properties override core-site configs.

[GitBox <gi...@apache.org>]

oleewere closed pull request #30: AMBARI-24833. Let logfeeder.properties override core-site configs.
URL: https://github.com/apache/ambari-logsearch/pull/30
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSS3UploadClient.java b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSS3UploadClient.java
index 5405a1051c..df1b8a55b2 100644
--- a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSS3UploadClient.java
+++ b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSS3UploadClient.java
@@ -57,6 +57,7 @@ public void init(LogFeederProps logFeederProps) {
     conf.set("fs.s3a.endpoint", s3OutputConfig.getEndpoint());
     conf.set("fs.s3a.path.style.access", String.valueOf(s3OutputConfig.isPathStyleAccess()));
     conf.set("fs.s3a.multiobjectdelete.enable", String.valueOf(s3OutputConfig.isMultiobjectDeleteEnable()));
+    LogFeederHDFSUtil.overrideFileSystemConfigs(logFeederProps, conf);
     this.fs = LogFeederHDFSUtil.buildFileSystem(conf);
   }
 
diff --git a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSUploadClient.java b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSUploadClient.java
index 421c4c5cb7..7e1b471de0 100644
--- a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSUploadClient.java
+++ b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSUploadClient.java
@@ -84,6 +84,7 @@ public void init(LogFeederProps logFeederProps) {
       }
     }
     logger.info("HDFS client - will use '{}' permission for uploaded files", hdfsOutputConfig.getHdfsFilePermissions());
+    LogFeederHDFSUtil.overrideFileSystemConfigs(logFeederProps, configuration);
     this.fs = LogFeederHDFSUtil.buildFileSystem(configuration);
   }
 
diff --git a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederHDFSUtil.java b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederHDFSUtil.java
index 3549e04af6..b741229a13 100644
--- a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederHDFSUtil.java
+++ b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederHDFSUtil.java
@@ -19,7 +19,10 @@
 package org.apache.ambari.logfeeder.util;
 
 import java.io.IOException;
+import java.util.Map;
+import java.util.Properties;
 
+import org.apache.ambari.logfeeder.conf.LogFeederProps;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -95,4 +98,20 @@ public static void closeFileSystem(FileSystem fileSystem) {
       }
     }
   }
+
+  /**
+   * Override Hadoop configuration object based on logfeeder.properties configurations (with keys that starts with "fs." or "hadoop.*")
+   * @param logFeederProps global property holder
+   * @param configuration hadoop configuration holder
+   */
+  public static void overrideFileSystemConfigs(LogFeederProps logFeederProps, Configuration configuration) {
+    Properties properties = logFeederProps.getProperties();
+    for (Map.Entry<Object, Object> prop : properties.entrySet()) {
+      String propertyName = prop.getKey().toString();
+      if (propertyName.startsWith("fs.")) {
+        logger.info("Override {} configuration (by logfeeder.properties)", propertyName);
+        configuration.set(propertyName, prop.getValue().toString());
+      }
+    }
+  }
 }
\ No newline at end of file
diff --git a/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties b/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties
index 45c05f39cf..2537913bbe 100644
--- a/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties
+++ b/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties
@@ -68,3 +68,5 @@ logfeeder.s3.access.key=MyAccessKey
 logfeeder.s3.object.acl=public-read
 logfeeder.s3.path.style.access=true
 logfeeder.s3.multiobject.delete.enable=false
+
+fs.s3a.acl.default=PublicReadWrite


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services