You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ka...@apache.org on 2020/06/30 17:40:11 UTC

[airflow] branch v1-10-test updated: Fix pickling failure when spawning processes (#8671)

This is an automated email from the ASF dual-hosted git repository.

kaxilnaik pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/v1-10-test by this push:
     new 834182e  Fix pickling failure when spawning processes (#8671)
834182e is described below

commit 834182ebe9f8271382f8a0c410c94f3db68c7b17
Author: jhtimmins <ja...@gmail.com>
AuthorDate: Wed May 6 02:43:28 2020 -0700

    Fix pickling failure when spawning processes (#8671)
    
    * Pull processor_factory out of _execute and move to the class scope.
    
    * Change the value of pickle_dags from True to False which is the default value of pickle_dags to be passed.
    
    * Fix how to reference FakeDagFileProcessorRunner class due to the place where it's defined is changed.
    
    * Fix configuration inheritance issue using  multiprocessing with spawn mode.
    * Add a new CI entry for spawn-multiprocessing method.
    
    * Add testcases for multiprocessing with spawn mode.
    
    Co-authored-by: Kousuke Saruta <sa...@oss.nttdata.com>
    Co-authored-by: James Timmins <ja...@astronomer.io>
    (cherry picked from commit 520aeedec82d91e15d5c43da4c85c948c5eb2ac3)
---
 airflow/configuration.py           |  57 ++++++++++------
 airflow/jobs/scheduler_job.py      |  33 ++++++---
 airflow/utils/dag_processing.py    |  36 ++++++++--
 airflow/utils/mixins.py            |  36 ++++++++++
 tests/jobs/test_scheduler_job.py   |  26 ++++++++
 tests/test_utils/mock_executor.py  |   9 ++-
 tests/utils/test_dag_processing.py | 133 +++++++++++++++++++++----------------
 7 files changed, 236 insertions(+), 94 deletions(-)

diff --git a/airflow/configuration.py b/airflow/configuration.py
index 5586de1..fe45a01 100644
--- a/airflow/configuration.py
+++ b/airflow/configuration.py
@@ -27,6 +27,7 @@ from collections import OrderedDict
 import copy
 import errno
 from future import standard_library
+import multiprocessing
 import os
 import shlex
 import six
@@ -197,6 +198,35 @@ class AirflowConfigParser(ConfigParser):
         self.is_validated = False
 
     def _validate(self):
+        self._validate_config_dependencies()
+        for section, replacement in self.deprecated_values.items():
+            for name, info in replacement.items():
+                old, new, version = info
+                if self.get(section, name, fallback=None) == old:
+                    # Make sure the env var option is removed, otherwise it
+                    # would be read and used instead of the value we set
+                    env_var = self._env_var_name(section, name)
+                    os.environ.pop(env_var, None)
+
+                    self.set(section, name, new)
+                    warnings.warn(
+                        'The {name} setting in [{section}] has the old default value '
+                        'of {old!r}. This value has been changed to {new!r} in the '
+                        'running config, but please update your config before Apache '
+                        'Airflow {version}.'.format(
+                            name=name, section=section, old=old, new=new, version=version
+                        ),
+                        FutureWarning
+                    )
+
+        self.is_validated = True
+
+    def _validate_config_dependencies(self):
+        """
+        Validate that config values aren't invalid given other config values
+        or system-level limitations and requirements.
+        """
+
         if (
                 self.get("core", "executor") not in ('DebugExecutor', 'SequentialExecutor') and
                 "sqlite" in self.get('core', 'sql_alchemy_conn')):
@@ -222,27 +252,14 @@ class AirflowConfigParser(ConfigParser):
                 "error: attempt at using ldapgroup "
                 "filtering without using the Ldap backend")
 
-        for section, replacement in self.deprecated_values.items():
-            for name, info in replacement.items():
-                old, new, version = info
-                if self.get(section, name, fallback=None) == old:
-                    # Make sure the env var option is removed, otherwise it
-                    # would be read and used instead of the value we set
-                    env_var = self._env_var_name(section, name)
-                    os.environ.pop(env_var, None)
-
-                    self.set(section, name, new)
-                    warnings.warn(
-                        'The {name} setting in [{section}] has the old default value '
-                        'of {old!r}. This value has been changed to {new!r} in the '
-                        'running config, but please update your config before Apache '
-                        'Airflow {version}.'.format(
-                            name=name, section=section, old=old, new=new, version=version
-                        ),
-                        FutureWarning
-                    )
+        if self.has_option('core', 'mp_start_method'):
+            mp_start_method = self.get('core', 'mp_start_method')
+            start_method_options = multiprocessing.get_all_start_methods()
 
-        self.is_validated = True
+            if mp_start_method not in start_method_options:
+                raise AirflowConfigException(
+                    "mp_start_method should not be " + mp_start_method +
+                    ". Possible values are " + ", ".join(start_method_options))
 
     @staticmethod
     def _env_var_name(section, key):
diff --git a/airflow/jobs/scheduler_job.py b/airflow/jobs/scheduler_job.py
index f290c57..97f3929 100644
--- a/airflow/jobs/scheduler_job.py
+++ b/airflow/jobs/scheduler_job.py
@@ -57,11 +57,12 @@ from airflow.utils.dag_processing import (AbstractDagFileProcessor,
                                           list_py_file_paths)
 from airflow.utils.db import provide_session
 from airflow.utils.email import get_email_address_list, send_email
+from airflow.utils.mixins import MultiprocessingStartMethodMixin
 from airflow.utils.log.logging_mixin import LoggingMixin, StreamLogWriter, set_context
 from airflow.utils.state import State
 
 
-class DagFileProcessor(AbstractDagFileProcessor, LoggingMixin):
+class DagFileProcessor(AbstractDagFileProcessor, LoggingMixin, MultiprocessingStartMethodMixin):
     """Helps call SchedulerJob.process_file() in a separate process.
 
     :param file_path: a Python file containing Airflow DAG definitions
@@ -177,8 +178,14 @@ class DagFileProcessor(AbstractDagFileProcessor, LoggingMixin):
         """
         Launch the process and start processing the DAG.
         """
-        self._parent_channel, _child_channel = multiprocessing.Pipe()
-        self._process = multiprocessing.Process(
+        if six.PY2:
+            context = multiprocessing
+        else:
+            start_method = self._get_multiprocessing_start_method()
+            context = multiprocessing.get_context(start_method)
+
+        self._parent_channel, _child_channel = context.Pipe()
+        self._process = context.Process(
             target=type(self)._run_file_processor,
             args=(
                 _child_channel,
@@ -1355,12 +1362,6 @@ class SchedulerJob(BaseJob):
         known_file_paths = list_py_file_paths(self.subdir)
         self.log.info("There are %s files in %s", len(known_file_paths), self.subdir)
 
-        def processor_factory(file_path, zombies):
-            return DagFileProcessor(file_path,
-                                    pickle_dags,
-                                    self.dag_ids,
-                                    zombies)
-
         # When using sqlite, we do not use async_mode
         # so the scheduler job and DAG parser don't access the DB at the same time.
         async_mode = not self.using_sqlite
@@ -1370,8 +1371,10 @@ class SchedulerJob(BaseJob):
         self.processor_agent = DagFileProcessorAgent(self.subdir,
                                                      known_file_paths,
                                                      self.num_runs,
-                                                     processor_factory,
+                                                     type(self)._create_dag_file_processor,
                                                      processor_timeout,
+                                                     self.dag_ids,
+                                                     pickle_dags,
                                                      async_mode)
 
         try:
@@ -1382,6 +1385,16 @@ class SchedulerJob(BaseJob):
             self.processor_agent.end()
             self.log.info("Exited execute loop")
 
+    @staticmethod
+    def _create_dag_file_processor(file_path, zombies, dag_ids, pickle_dags):
+        """
+        Creates DagFileProcessorProcess instance.
+        """
+        return DagFileProcessor(file_path,
+                                pickle_dags,
+                                dag_ids,
+                                zombies)
+
     def _get_simple_dags(self):
         return self.processor_agent.harvest_simple_dags()
 
diff --git a/airflow/utils/dag_processing.py b/airflow/utils/dag_processing.py
index e4ecc29..d13b192 100644
--- a/airflow/utils/dag_processing.py
+++ b/airflow/utils/dag_processing.py
@@ -55,6 +55,7 @@ from airflow.utils import timezone
 from airflow.utils.helpers import reap_process_group
 from airflow.utils.db import provide_session
 from airflow.utils.log.logging_mixin import LoggingMixin
+from airflow.utils.mixins import MultiprocessingStartMethodMixin
 from airflow.utils.state import State
 
 if six.PY2:
@@ -473,7 +474,7 @@ class DagParsingSignal(enum.Enum):
     END_MANAGER = 'end_manager'
 
 
-class DagFileProcessorAgent(LoggingMixin):
+class DagFileProcessorAgent(LoggingMixin, MultiprocessingStartMethodMixin):
     """
     Agent for DAG file processing. It is responsible for all DAG parsing
     related jobs in scheduler process. Mainly it can spin up DagFileProcessorManager
@@ -489,6 +490,8 @@ class DagFileProcessorAgent(LoggingMixin):
                  max_runs,
                  processor_factory,
                  processor_timeout,
+                 dag_ids,
+                 pickle_dags,
                  async_mode):
         """
         :param dag_directory: Directory where DAG definitions are kept. All
@@ -513,6 +516,8 @@ class DagFileProcessorAgent(LoggingMixin):
         self._max_runs = max_runs
         self._processor_factory = processor_factory
         self._processor_timeout = processor_timeout
+        self._dag_ids = dag_ids
+        self._pickle_dags = pickle_dags
         self._async_mode = async_mode
         # Map from file path to the processor
         self._processors = {}
@@ -529,8 +534,14 @@ class DagFileProcessorAgent(LoggingMixin):
         """
         Launch DagFileProcessorManager processor and start DAG parsing loop in manager.
         """
-        self._parent_signal_conn, child_signal_conn = multiprocessing.Pipe()
-        self._process = multiprocessing.Process(
+        if six.PY2:
+            context = multiprocessing
+        else:
+            mp_start_method = self._get_multiprocessing_start_method()
+            context = multiprocessing.get_context(mp_start_method)
+
+        self._parent_signal_conn, child_signal_conn = context.Pipe()
+        self._process = context.Process(
             target=type(self)._run_processor_manager,
             args=(
                 self._dag_directory,
@@ -539,6 +550,8 @@ class DagFileProcessorAgent(LoggingMixin):
                 self._processor_factory,
                 self._processor_timeout,
                 child_signal_conn,
+                self._dag_ids,
+                self._pickle_dags,
                 self._async_mode,
             )
         )
@@ -584,6 +597,8 @@ class DagFileProcessorAgent(LoggingMixin):
                                processor_factory,
                                processor_timeout,
                                signal_conn,
+                               dag_ids,
+                               pickle_dags,
                                async_mode):
 
         # Make this process start as a new process group - that makes it easy
@@ -610,6 +625,8 @@ class DagFileProcessorAgent(LoggingMixin):
                                                     processor_factory,
                                                     processor_timeout,
                                                     signal_conn,
+                                                    dag_ids,
+                                                    pickle_dags,
                                                     async_mode)
 
         processor_manager.start()
@@ -722,6 +739,10 @@ class DagFileProcessorManager(LoggingMixin):
     :type processor_timeout: timedelta
     :param signal_conn: connection to communicate signal with processor agent.
     :type signal_conn: airflow.models.connection.Connection
+    :param dag_ids: if specified, only schedule tasks with these DAG IDs
+    :type dag_ids: list[str]
+    :param pickle_dags: whether to pickle DAGs.
+    :type pickle_dags: bool
     :param async_mode: whether to start the manager in async mode
     :type async_mode: bool
     """
@@ -733,6 +754,8 @@ class DagFileProcessorManager(LoggingMixin):
                  processor_factory,
                  processor_timeout,
                  signal_conn,
+                 dag_ids,
+                 pickle_dags,
                  async_mode=True):
         self._file_paths = file_paths
         self._file_path_queue = []
@@ -740,6 +763,8 @@ class DagFileProcessorManager(LoggingMixin):
         self._max_runs = max_runs
         self._processor_factory = processor_factory
         self._signal_conn = signal_conn
+        self._pickle_dags = pickle_dags
+        self._dag_ids = dag_ids
         self._async_mode = async_mode
 
         self._parallelism = conf.getint('scheduler', 'max_threads')
@@ -1216,7 +1241,8 @@ class DagFileProcessorManager(LoggingMixin):
         """
         while self._parallelism - len(self._processors) > 0 and self._file_path_queue:
             file_path = self._file_path_queue.pop(0)
-            processor = self._processor_factory(file_path, self._zombies)
+            processor = self._processor_factory(file_path, self._zombies,
+                                                self._dag_ids, self._pickle_dags)
             Stats.incr('dag_processing.processes')
 
             processor.start()
@@ -1280,7 +1306,7 @@ class DagFileProcessorManager(LoggingMixin):
         if not self._last_zombie_query_time or \
                 (now - self._last_zombie_query_time).total_seconds() > self._zombie_query_interval:
             # to avoid circular imports
-            from airflow.jobs import LocalTaskJob as LJ
+            from airflow.jobs.local_task_job import LocalTaskJob as LJ
             self.log.info("Finding 'running' jobs without a recent heartbeat")
             TI = airflow.models.TaskInstance
             limit_dttm = timezone.utcnow() - timedelta(
diff --git a/airflow/utils/mixins.py b/airflow/utils/mixins.py
new file mode 100644
index 0000000..9bbc1fe
--- /dev/null
+++ b/airflow/utils/mixins.py
@@ -0,0 +1,36 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import multiprocessing
+
+from airflow.configuration import conf
+
+
+class MultiprocessingStartMethodMixin:
+    """
+    Convenience class to add support for different types of multiprocessing.
+    """
+    def _get_multiprocessing_start_method(self):
+        """
+        Determine method of creating new processes by checking if the
+        mp_start_method is set in configs, else, it uses the OS default.
+        """
+        if conf.has_option('core', 'mp_start_method'):
+            return conf.get('core', 'mp_start_method')
+
+        return multiprocessing.get_start_method()
diff --git a/tests/jobs/test_scheduler_job.py b/tests/jobs/test_scheduler_job.py
index 181d193..d636fd0 100644
--- a/tests/jobs/test_scheduler_job.py
+++ b/tests/jobs/test_scheduler_job.py
@@ -55,6 +55,7 @@ from airflow.utils.db import create_session, provide_session
 from airflow.utils.state import State
 from tests.compat import MagicMock, Mock, PropertyMock, mock, patch
 from tests.test_core import TEST_DAG_FOLDER
+from tests.test_utils.config import conf_vars
 from tests.test_utils.db import (
     clear_db_dags, clear_db_errors, clear_db_pools, clear_db_runs, clear_db_sla_miss, set_default_pool_slots,
 )
@@ -1377,6 +1378,31 @@ class SchedulerJobTest(unittest.TestCase):
         self.assertEqual(
             len(session.query(TI).filter(TI.dag_id == dag_id).all()), 0)
 
+    @conf_vars({("core", "mp_start_method"): "spawn"})
+    def test_scheduler_multiprocessing_with_spawn_method(self):
+        """
+        Test that the scheduler can successfully queue multiple dags in parallel
+        when using "spawn" mode of multiprocessing. (Fork is default on Linux and older OSX)
+        """
+        dag_ids = ['test_start_date_scheduling', 'test_dagrun_states_success']
+        for dag_id in dag_ids:
+            dag = self.dagbag.get_dag(dag_id)
+            dag.clear()
+
+        scheduler = SchedulerJob(dag_ids=dag_ids,
+                                 executor=self.null_exec,
+                                 subdir=os.path.join(
+                                     TEST_DAG_FOLDER, 'test_scheduler_dags.py'),
+                                 num_runs=1)
+
+        scheduler.run()
+
+        # zero tasks ran
+        dag_id = 'test_start_date_scheduling'
+        with create_session() as session:
+            self.assertEqual(
+                session.query(TaskInstance).filter(TaskInstance.dag_id == dag_id).count(), 0)
+
     def test_scheduler_dagrun_once(self):
         """
         Test if the scheduler does not create multiple dagruns
diff --git a/tests/test_utils/mock_executor.py b/tests/test_utils/mock_executor.py
index eb64b3b..a266682 100644
--- a/tests/test_utils/mock_executor.py
+++ b/tests/test_utils/mock_executor.py
@@ -37,10 +37,17 @@ class MockExecutor(BaseExecutor):
         self.history = []
         # All the tasks, in a stable sort order
         self.sorted_tasks = []
-        self.mock_task_results = defaultdict(lambda: State.SUCCESS)
+
+        # If multiprocessing runs in spawn mode,
+        # arguments are to be pickled but lambda is not picclable.
+        # So we should pass self.success instead of lambda.
+        self.mock_task_results = defaultdict(self.success)
 
         super(MockExecutor, self).__init__(*args, **kwargs)
 
+    def success(self):
+        return State.SUCCESS
+
     def heartbeat(self):
         if not self.do_update:
             return
diff --git a/tests/utils/test_dag_processing.py b/tests/utils/test_dag_processing.py
index 2232310..3726a8d 100644
--- a/tests/utils/test_dag_processing.py
+++ b/tests/utils/test_dag_processing.py
@@ -135,6 +135,42 @@ class settings_context(object):
         sys.path.remove(self.settings_root)
 
 
+class FakeDagFileProcessorRunner(DagFileProcessor):
+    # This fake processor will return the zombies it received in constructor
+    # as its processing result w/o actually parsing anything.
+    def __init__(self, file_path, pickle_dags, dag_id_allow_list, zombies):
+        super(FakeDagFileProcessorRunner, self).__init__(file_path, pickle_dags, dag_id_allow_list, zombies)
+        self._result = zombies, 0
+
+    def start(self):
+        pass
+
+    @property
+    def start_time(self):
+        return DEFAULT_DATE
+
+    @property
+    def pid(self):
+        return 1234
+
+    @property
+    def done(self):
+        return True
+
+    @property
+    def result(self):
+        return self._result
+
+    @staticmethod
+    def _fake_dag_processor_factory(file_path, zombies, dag_ids, pickle_dags):
+        return FakeDagFileProcessorRunner(
+            file_path,
+            pickle_dags,
+            dag_ids,
+            zombies
+        )
+
+
 class TestDagFileProcessorManager(unittest.TestCase):
     def setUp(self):
         clear_db_runs()
@@ -147,6 +183,8 @@ class TestDagFileProcessorManager(unittest.TestCase):
             processor_factory=MagicMock().return_value,
             processor_timeout=timedelta.max,
             signal_conn=MagicMock(),
+            dag_ids=[],
+            pickle_dags=False,
             async_mode=True)
 
         mock_processor = MagicMock()
@@ -168,6 +206,8 @@ class TestDagFileProcessorManager(unittest.TestCase):
             processor_factory=MagicMock().return_value,
             processor_timeout=timedelta.max,
             signal_conn=MagicMock(),
+            dag_ids=[],
+            pickle_dags=False,
             async_mode=True)
 
         mock_processor = MagicMock()
@@ -188,6 +228,8 @@ class TestDagFileProcessorManager(unittest.TestCase):
             processor_factory=MagicMock().return_value,
             processor_timeout=timedelta.max,
             signal_conn=MagicMock(),
+            dag_ids=[],
+            pickle_dags=False,
             async_mode=True)
 
         dagbag = DagBag(TEST_DAG_FOLDER)
@@ -224,12 +266,14 @@ class TestDagFileProcessorManager(unittest.TestCase):
         Check that the same set of zombies are passed to the dag
         file processors until the next zombie detection logic is invoked.
         """
+        test_dag_path = os.path.join(TEST_DAG_FOLDER, 'test_example_bash_operator.py')
         with conf_vars({('scheduler', 'max_threads'): '1',
                         ('core', 'load_examples'): 'False'}):
-            dagbag = DagBag(os.path.join(TEST_DAG_FOLDER, 'test_example_bash_operator.py'))
+            dagbag = DagBag(test_dag_path)
             with create_session() as session:
                 session.query(LJ).delete()
                 dag = dagbag.get_dag('test_example_bash_operator')
+                dag.sync_to_db()
                 task = dag.get_task(task_id='run_this_last')
 
                 ti = TI(task, DEFAULT_DATE, State.RUNNING)
@@ -243,49 +287,14 @@ class TestDagFileProcessorManager(unittest.TestCase):
                 session.commit()
                 fake_zombies = [SimpleTaskInstance(ti)]
 
-            class FakeDagFIleProcessor(DagFileProcessor):
-                # This fake processor will return the zombies it received in constructor
-                # as its processing result w/o actually parsing anything.
-                def __init__(self, file_path, pickle_dags, dag_ids, zombies):
-                    super(FakeDagFIleProcessor, self).__init__(
-                        file_path, pickle_dags, dag_ids, zombies
-                    )
-
-                    self._result = zombies, 0
-
-                def start(self):
-                    pass
-
-                @property
-                def start_time(self):
-                    return DEFAULT_DATE
-
-                @property
-                def pid(self):
-                    return 1234
-
-                @property
-                def done(self):
-                    return True
-
-                @property
-                def result(self):
-                    return self._result
-
-            def processor_factory(file_path, zombies):
-                return FakeDagFIleProcessor(file_path,
-                                            False,
-                                            [],
-                                            zombies)
-
-            test_dag_path = os.path.join(TEST_DAG_FOLDER,
-                                         'test_example_bash_operator.py')
             async_mode = 'sqlite' not in conf.get('core', 'sql_alchemy_conn')
             processor_agent = DagFileProcessorAgent(test_dag_path,
                                                     [],
                                                     1,
-                                                    processor_factory,
+                                                    FakeDagFileProcessorRunner._fake_dag_processor_factory,
                                                     timedelta.max,
+                                                    [],
+                                                    False,
                                                     async_mode)
             processor_agent.start()
             parsing_result = []
@@ -311,6 +320,8 @@ class TestDagFileProcessorManager(unittest.TestCase):
             processor_factory=MagicMock().return_value,
             processor_timeout=timedelta(seconds=5),
             signal_conn=MagicMock(),
+            dag_ids=[],
+            pickle_dags=False,
             async_mode=True)
 
         processor = DagFileProcessor('abc.txt', False, [], [])
@@ -330,6 +341,8 @@ class TestDagFileProcessorManager(unittest.TestCase):
             processor_factory=MagicMock().return_value,
             processor_timeout=timedelta(seconds=5),
             signal_conn=MagicMock(),
+            dag_ids=[],
+            pickle_dags=False,
             async_mode=True)
 
         processor = DagFileProcessor('abc.txt', False, [], [])
@@ -347,8 +360,19 @@ class TestDagFileProcessorAgent(unittest.TestCase):
     def tearDown(self):
         # Remove any new modules imported during the test run. This lets us
         # import the same source files for more than one test.
+        remove_list = []
         for m in [m for m in sys.modules if m not in self.old_modules]:
-            del sys.modules[m]
+            remove_list.append(m)
+
+        for mod in remove_list:
+            del sys.modules[mod]
+
+    @staticmethod
+    def _processor_factory(file_path, zombies, dag_ids, pickle_dags):
+        return DagFileProcessor(file_path,
+                                pickle_dags,
+                                dag_ids,
+                                zombies)
 
     def test_reload_module(self):
         """
@@ -359,16 +383,12 @@ class TestDagFileProcessorAgent(unittest.TestCase):
         with settings_context(SETTINGS_FILE_VALID):
             # Launch a process through DagFileProcessorAgent, which will try
             # reload the logging module.
-            def processor_factory(file_path, zombies):
-                return DagFileProcessor(file_path,
-                                        False,
-                                        [],
-                                        zombies)
 
             test_dag_path = os.path.join(TEST_DAG_FOLDER, 'test_scheduler_dags.py')
             async_mode = 'sqlite' not in conf.get('core', 'sql_alchemy_conn')
 
             log_file_loc = conf.get('core', 'DAG_PROCESSOR_MANAGER_LOG_LOCATION')
+
             try:
                 os.remove(log_file_loc)
             except OSError:
@@ -378,8 +398,10 @@ class TestDagFileProcessorAgent(unittest.TestCase):
             processor_agent = DagFileProcessorAgent(test_dag_path,
                                                     [],
                                                     0,
-                                                    processor_factory,
+                                                    type(self)._processor_factory,
                                                     timedelta.max,
+                                                    [],
+                                                    False,
                                                     async_mode)
             processor_agent.start()
             if not async_mode:
@@ -389,22 +411,20 @@ class TestDagFileProcessorAgent(unittest.TestCase):
 
             # Since we are reloading logging config not creating this file,
             # we should expect it to be nonexistent.
+
             self.assertFalse(os.path.isfile(log_file_loc))
 
     def test_parse_once(self):
-        def processor_factory(file_path, zombies):
-            return DagFileProcessor(file_path,
-                                    False,
-                                    [],
-                                    zombies)
 
         test_dag_path = os.path.join(TEST_DAG_FOLDER, 'test_scheduler_dags.py')
         async_mode = 'sqlite' not in conf.get('core', 'sql_alchemy_conn')
         processor_agent = DagFileProcessorAgent(test_dag_path,
                                                 [test_dag_path],
                                                 1,
-                                                processor_factory,
+                                                type(self)._processor_factory,
                                                 timedelta.max,
+                                                [],
+                                                False,
                                                 async_mode)
         processor_agent.start()
         parsing_result = []
@@ -419,11 +439,6 @@ class TestDagFileProcessorAgent(unittest.TestCase):
         self.assertEqual(dag_ids.count('test_start_date_scheduling'), 1)
 
     def test_launch_process(self):
-        def processor_factory(file_path, zombies):
-            return DagFileProcessor(file_path,
-                                    False,
-                                    [],
-                                    zombies)
 
         test_dag_path = os.path.join(TEST_DAG_FOLDER, 'test_scheduler_dags.py')
         async_mode = 'sqlite' not in conf.get('core', 'sql_alchemy_conn')
@@ -438,8 +453,10 @@ class TestDagFileProcessorAgent(unittest.TestCase):
         processor_agent = DagFileProcessorAgent(test_dag_path,
                                                 [],
                                                 0,
-                                                processor_factory,
+                                                type(self)._processor_factory,
                                                 timedelta.max,
+                                                [],
+                                                False,
                                                 async_mode)
         processor_agent.start()
         if not async_mode: