[incubator-heron] branch master updated: Bug: Read override config file in heron executor (#3292)

[nw...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

nwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git


The following commit(s) were added to refs/heads/master by this push:
     new d7f7f3e  Bug: Read override config file in heron executor (#3292)
d7f7f3e is described below

commit d7f7f3e6e8285ee0d0a2f5c98af8a8c6f262d2d1
Author: Rohan Agarwal <ro...@gmail.com>
AuthorDate: Tue Jun 18 17:01:34 2019 -0700

    Bug: Read override config file in heron executor (#3292)
    
    * read override config file in heron executor
    
    * retrigger ci for flaky integration test
    
    * check for None
---
 heron/executor/src/python/heron_executor.py | 9 ++++++++-
 heron/statemgrs/src/python/configloader.py  | 5 ++---
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/heron/executor/src/python/heron_executor.py b/heron/executor/src/python/heron_executor.py
index 0c8f21c..6f5790a 100755
--- a/heron/executor/src/python/heron_executor.py
+++ b/heron/executor/src/python/heron_executor.py
@@ -1109,10 +1109,17 @@ class HeronExecutor(object):
     Receive updates to the packing plan from the statemgrs and update processes as needed.
     """
     Log.info("Start state manager watches")
+
+    with open(self.override_config_file, 'r') as stream:
+      overrides = yaml.load(stream)
+      if overrides is None:
+        overrides = dict()
+    overrides["heron.statemgr.connection.string"] = self.state_manager_connection
+
     statemgr_config = StateMgrConfig()
     statemgr_config.set_state_locations(configloader.load_state_manager_locations(
         self.cluster, state_manager_config_file=self.state_manager_config_file,
-        overrides={"heron.statemgr.connection.string": self.state_manager_connection}))
+        **overrides))
     try:
       self.state_managers = statemanagerfactory.get_all_state_managers(statemgr_config)
       for state_manager in self.state_managers:
diff --git a/heron/statemgrs/src/python/configloader.py b/heron/statemgrs/src/python/configloader.py
index 338a71d..384a5c7 100644
--- a/heron/statemgrs/src/python/configloader.py
+++ b/heron/statemgrs/src/python/configloader.py
@@ -28,7 +28,7 @@ import yaml
 # pylint: disable=dangerous-default-value
 
 def load_state_manager_locations(cluster, state_manager_config_file='heron-conf/statemgr.yaml',
-                                 overrides={}):
+                                 **overrides):
   """ Reads configs to determine which state manager to use and converts them to state manager
   locations. Handles a subset of config wildcard substitution supported in the substitute method in
   org.apache.heron.spi.common.Misc.java"""
@@ -47,8 +47,7 @@ def load_state_manager_locations(cluster, state_manager_config_file='heron-conf/
   config = __replace(config, wildcards, state_manager_config_file)
 
   # merge with overrides
-  if overrides:
-    config.update(overrides)
+  config.update(overrides)
 
   # need to convert from the format in statemgr.yaml to the format that the python state managers
   # takes. first, set reasonable defaults to local