[gobblin] branch master updated: [GOBBLIN-1635] Avoid loading env configuration when using config store to improve the performance (#3496)

[zi...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

zihanli58 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/gobblin.git


The following commit(s) were added to refs/heads/master by this push:
     new 065027891 [GOBBLIN-1635] Avoid loading env configuration when using config store to improve the performance (#3496)
065027891 is described below

commit 065027891e137dc18af8d1ac8b6a9680682774ea
Author: Zihan Li <zi...@linkedin.com>
AuthorDate: Wed Apr 27 14:28:43 2022 -0700

    [GOBBLIN-1635] Avoid loading env configuration when using config store to improve the performance (#3496)
    
    * address comments
    
    * use connectionmanager when httpclient is not cloesable
    
    * [GOBBLIN-1631]Emit heartbeat for dagManagerThread
    
    * [GOBBLIN-1635] Avoid loading env configuration when using config store to improve the performance
    
    * [GOBBLIN-1630] Remove flow level metrics for adhoc flows (#3491)
    
    * Remove emitting metrics for adhoc flows in dagmanager and orchestrator
    
    * Add tests
    
    * Fix tests
    
    * Address comments
    
    * Improve test by validating gauge value
    
    * use data node aliases to figure out data node names before using DMAS (#3493)
    
    * [GOBBLIN-1619] WriterUtils.mkdirsWithRecursivePermission contains race condition and puts unnecessary load on filesystem (#3477)
    
    * [GOBBLIN-1619] Fix race cond. in writerutil mkdirs
    
    * writer util mkdirs previously had race condition when multiple processes
    try to create the same parent directory. This causes incorrect
    FileNotFoundException
    * new implementation does not change the behavior
    
    * Test coverage for retry config
    
    * Wait for file to exist via retry cfg before setting perms
    
    * use user supplied props to create FileSystem in DatasetCleanerTask (#3483)
    
    Co-authored-by: Zihan Li <zi...@zihli-mn2.linkedin.biz>
    Co-authored-by: William Lo <lo...@gmail.com>
    Co-authored-by: Arjun Singh Bora <ab...@linkedin.com>
    Co-authored-by: Matthew Ho <ho...@gmail.com>
---
 .../gobblin/config/common/impl/ConfigStoreBackedValueInspector.java   | 2 +-
 .../gobblin/config/store/hdfs/SimpleHadoopFilesystemConfigStore.java  | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/common/impl/ConfigStoreBackedValueInspector.java b/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/common/impl/ConfigStoreBackedValueInspector.java
index c7e40bd3d..2dc0bfb85 100644
--- a/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/common/impl/ConfigStoreBackedValueInspector.java
+++ b/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/common/impl/ConfigStoreBackedValueInspector.java
@@ -153,7 +153,7 @@ public class ConfigStoreBackedValueInspector implements ConfigStoreValueInspecto
    */
   public Config getResolvedConfig(ConfigKeyPath configKey, Optional<Config> runtimeConfig) {
     return getResolvedConfigRecursive(configKey, Sets.<String>newHashSet(), runtimeConfig)
-        .withFallback(ConfigFactory.defaultOverrides()).withFallback(ConfigFactory.systemEnvironment()).resolve();
+        .withFallback(ConfigFactory.defaultOverrides()).resolve();
   }
 
   @Override
diff --git a/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/hdfs/SimpleHadoopFilesystemConfigStore.java b/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/hdfs/SimpleHadoopFilesystemConfigStore.java
index 4008065dc..1fb4f6c41 100644
--- a/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/hdfs/SimpleHadoopFilesystemConfigStore.java
+++ b/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/hdfs/SimpleHadoopFilesystemConfigStore.java
@@ -314,11 +314,11 @@ public class SimpleHadoopFilesystemConfigStore implements ConfigStore, Deployabl
     if (includesBuilder.length() > 0) {
       if (runtimeConfig.isPresent()) {
         return ConfigFactory.parseString(includesBuilder.toString()).withFallback(ConfigFactory.defaultOverrides())
-            .withFallback(ConfigFactory.systemEnvironment()).withFallback(runtimeConfig.get()).resolve()
+            .withFallback(runtimeConfig.get()).resolve()
             .getStringList(INCLUDES_KEY_NAME);
       } else {
         return ConfigFactory.parseString(includesBuilder.toString()).withFallback(ConfigFactory.defaultOverrides())
-            .withFallback(ConfigFactory.systemEnvironment()).resolve().getStringList(INCLUDES_KEY_NAME);
+            .resolve().getStringList(INCLUDES_KEY_NAME);
       }
     }