You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ka...@apache.org on 2020/06/30 17:40:11 UTC
[airflow] branch v1-10-test updated: Fix pickling failure when
spawning processes (#8671)
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v1-10-test by this push:
new 834182e Fix pickling failure when spawning processes (#8671)
834182e is described below
commit 834182ebe9f8271382f8a0c410c94f3db68c7b17
Author: jhtimmins <ja...@gmail.com>
AuthorDate: Wed May 6 02:43:28 2020 -0700
Fix pickling failure when spawning processes (#8671)
* Pull processor_factory out of _execute and move to the class scope.
* Change the value of pickle_dags from True to False which is the default value of pickle_dags to be passed.
* Fix how to reference FakeDagFileProcessorRunner class due to the place where it's defined is changed.
* Fix configuration inheritance issue using multiprocessing with spawn mode.
* Add a new CI entry for spawn-multiprocessing method.
* Add testcases for multiprocessing with spawn mode.
Co-authored-by: Kousuke Saruta <sa...@oss.nttdata.com>
Co-authored-by: James Timmins <ja...@astronomer.io>
(cherry picked from commit 520aeedec82d91e15d5c43da4c85c948c5eb2ac3)
---
airflow/configuration.py | 57 ++++++++++------
airflow/jobs/scheduler_job.py | 33 ++++++---
airflow/utils/dag_processing.py | 36 ++++++++--
airflow/utils/mixins.py | 36 ++++++++++
tests/jobs/test_scheduler_job.py | 26 ++++++++
tests/test_utils/mock_executor.py | 9 ++-
tests/utils/test_dag_processing.py | 133 +++++++++++++++++++++----------------
7 files changed, 236 insertions(+), 94 deletions(-)
diff --git a/airflow/configuration.py b/airflow/configuration.py
index 5586de1..fe45a01 100644
--- a/airflow/configuration.py
+++ b/airflow/configuration.py
@@ -27,6 +27,7 @@ from collections import OrderedDict
import copy
import errno
from future import standard_library
+import multiprocessing
import os
import shlex
import six
@@ -197,6 +198,35 @@ class AirflowConfigParser(ConfigParser):
self.is_validated = False
def _validate(self):
+ self._validate_config_dependencies()
+ for section, replacement in self.deprecated_values.items():
+ for name, info in replacement.items():
+ old, new, version = info
+ if self.get(section, name, fallback=None) == old:
+ # Make sure the env var option is removed, otherwise it
+ # would be read and used instead of the value we set
+ env_var = self._env_var_name(section, name)
+ os.environ.pop(env_var, None)
+
+ self.set(section, name, new)
+ warnings.warn(
+ 'The {name} setting in [{section}] has the old default value '
+ 'of {old!r}. This value has been changed to {new!r} in the '
+ 'running config, but please update your config before Apache '
+ 'Airflow {version}.'.format(
+ name=name, section=section, old=old, new=new, version=version
+ ),
+ FutureWarning
+ )
+
+ self.is_validated = True
+
+ def _validate_config_dependencies(self):
+ """
+ Validate that config values aren't invalid given other config values
+ or system-level limitations and requirements.
+ """
+
if (
self.get("core", "executor") not in ('DebugExecutor', 'SequentialExecutor') and
"sqlite" in self.get('core', 'sql_alchemy_conn')):
@@ -222,27 +252,14 @@ class AirflowConfigParser(ConfigParser):
"error: attempt at using ldapgroup "
"filtering without using the Ldap backend")
- for section, replacement in self.deprecated_values.items():
- for name, info in replacement.items():
- old, new, version = info
- if self.get(section, name, fallback=None) == old:
- # Make sure the env var option is removed, otherwise it
- # would be read and used instead of the value we set
- env_var = self._env_var_name(section, name)
- os.environ.pop(env_var, None)
-
- self.set(section, name, new)
- warnings.warn(
- 'The {name} setting in [{section}] has the old default value '
- 'of {old!r}. This value has been changed to {new!r} in the '
- 'running config, but please update your config before Apache '
- 'Airflow {version}.'.format(
- name=name, section=section, old=old, new=new, version=version
- ),
- FutureWarning
- )
+ if self.has_option('core', 'mp_start_method'):
+ mp_start_method = self.get('core', 'mp_start_method')
+ start_method_options = multiprocessing.get_all_start_methods()
- self.is_validated = True
+ if mp_start_method not in start_method_options:
+ raise AirflowConfigException(
+ "mp_start_method should not be " + mp_start_method +
+ ". Possible values are " + ", ".join(start_method_options))
@staticmethod
def _env_var_name(section, key):
diff --git a/airflow/jobs/scheduler_job.py b/airflow/jobs/scheduler_job.py
index f290c57..97f3929 100644
--- a/airflow/jobs/scheduler_job.py
+++ b/airflow/jobs/scheduler_job.py
@@ -57,11 +57,12 @@ from airflow.utils.dag_processing import (AbstractDagFileProcessor,
list_py_file_paths)
from airflow.utils.db import provide_session
from airflow.utils.email import get_email_address_list, send_email
+from airflow.utils.mixins import MultiprocessingStartMethodMixin
from airflow.utils.log.logging_mixin import LoggingMixin, StreamLogWriter, set_context
from airflow.utils.state import State
-class DagFileProcessor(AbstractDagFileProcessor, LoggingMixin):
+class DagFileProcessor(AbstractDagFileProcessor, LoggingMixin, MultiprocessingStartMethodMixin):
"""Helps call SchedulerJob.process_file() in a separate process.
:param file_path: a Python file containing Airflow DAG definitions
@@ -177,8 +178,14 @@ class DagFileProcessor(AbstractDagFileProcessor, LoggingMixin):
"""
Launch the process and start processing the DAG.
"""
- self._parent_channel, _child_channel = multiprocessing.Pipe()
- self._process = multiprocessing.Process(
+ if six.PY2:
+ context = multiprocessing
+ else:
+ start_method = self._get_multiprocessing_start_method()
+ context = multiprocessing.get_context(start_method)
+
+ self._parent_channel, _child_channel = context.Pipe()
+ self._process = context.Process(
target=type(self)._run_file_processor,
args=(
_child_channel,
@@ -1355,12 +1362,6 @@ class SchedulerJob(BaseJob):
known_file_paths = list_py_file_paths(self.subdir)
self.log.info("There are %s files in %s", len(known_file_paths), self.subdir)
- def processor_factory(file_path, zombies):
- return DagFileProcessor(file_path,
- pickle_dags,
- self.dag_ids,
- zombies)
-
# When using sqlite, we do not use async_mode
# so the scheduler job and DAG parser don't access the DB at the same time.
async_mode = not self.using_sqlite
@@ -1370,8 +1371,10 @@ class SchedulerJob(BaseJob):
self.processor_agent = DagFileProcessorAgent(self.subdir,
known_file_paths,
self.num_runs,
- processor_factory,
+ type(self)._create_dag_file_processor,
processor_timeout,
+ self.dag_ids,
+ pickle_dags,
async_mode)
try:
@@ -1382,6 +1385,16 @@ class SchedulerJob(BaseJob):
self.processor_agent.end()
self.log.info("Exited execute loop")
+ @staticmethod
+ def _create_dag_file_processor(file_path, zombies, dag_ids, pickle_dags):
+ """
+ Creates DagFileProcessorProcess instance.
+ """
+ return DagFileProcessor(file_path,
+ pickle_dags,
+ dag_ids,
+ zombies)
+
def _get_simple_dags(self):
return self.processor_agent.harvest_simple_dags()
diff --git a/airflow/utils/dag_processing.py b/airflow/utils/dag_processing.py
index e4ecc29..d13b192 100644
--- a/airflow/utils/dag_processing.py
+++ b/airflow/utils/dag_processing.py
@@ -55,6 +55,7 @@ from airflow.utils import timezone
from airflow.utils.helpers import reap_process_group
from airflow.utils.db import provide_session
from airflow.utils.log.logging_mixin import LoggingMixin
+from airflow.utils.mixins import MultiprocessingStartMethodMixin
from airflow.utils.state import State
if six.PY2:
@@ -473,7 +474,7 @@ class DagParsingSignal(enum.Enum):
END_MANAGER = 'end_manager'
-class DagFileProcessorAgent(LoggingMixin):
+class DagFileProcessorAgent(LoggingMixin, MultiprocessingStartMethodMixin):
"""
Agent for DAG file processing. It is responsible for all DAG parsing
related jobs in scheduler process. Mainly it can spin up DagFileProcessorManager
@@ -489,6 +490,8 @@ class DagFileProcessorAgent(LoggingMixin):
max_runs,
processor_factory,
processor_timeout,
+ dag_ids,
+ pickle_dags,
async_mode):
"""
:param dag_directory: Directory where DAG definitions are kept. All
@@ -513,6 +516,8 @@ class DagFileProcessorAgent(LoggingMixin):
self._max_runs = max_runs
self._processor_factory = processor_factory
self._processor_timeout = processor_timeout
+ self._dag_ids = dag_ids
+ self._pickle_dags = pickle_dags
self._async_mode = async_mode
# Map from file path to the processor
self._processors = {}
@@ -529,8 +534,14 @@ class DagFileProcessorAgent(LoggingMixin):
"""
Launch DagFileProcessorManager processor and start DAG parsing loop in manager.
"""
- self._parent_signal_conn, child_signal_conn = multiprocessing.Pipe()
- self._process = multiprocessing.Process(
+ if six.PY2:
+ context = multiprocessing
+ else:
+ mp_start_method = self._get_multiprocessing_start_method()
+ context = multiprocessing.get_context(mp_start_method)
+
+ self._parent_signal_conn, child_signal_conn = context.Pipe()
+ self._process = context.Process(
target=type(self)._run_processor_manager,
args=(
self._dag_directory,
@@ -539,6 +550,8 @@ class DagFileProcessorAgent(LoggingMixin):
self._processor_factory,
self._processor_timeout,
child_signal_conn,
+ self._dag_ids,
+ self._pickle_dags,
self._async_mode,
)
)
@@ -584,6 +597,8 @@ class DagFileProcessorAgent(LoggingMixin):
processor_factory,
processor_timeout,
signal_conn,
+ dag_ids,
+ pickle_dags,
async_mode):
# Make this process start as a new process group - that makes it easy
@@ -610,6 +625,8 @@ class DagFileProcessorAgent(LoggingMixin):
processor_factory,
processor_timeout,
signal_conn,
+ dag_ids,
+ pickle_dags,
async_mode)
processor_manager.start()
@@ -722,6 +739,10 @@ class DagFileProcessorManager(LoggingMixin):
:type processor_timeout: timedelta
:param signal_conn: connection to communicate signal with processor agent.
:type signal_conn: airflow.models.connection.Connection
+ :param dag_ids: if specified, only schedule tasks with these DAG IDs
+ :type dag_ids: list[str]
+ :param pickle_dags: whether to pickle DAGs.
+ :type pickle_dags: bool
:param async_mode: whether to start the manager in async mode
:type async_mode: bool
"""
@@ -733,6 +754,8 @@ class DagFileProcessorManager(LoggingMixin):
processor_factory,
processor_timeout,
signal_conn,
+ dag_ids,
+ pickle_dags,
async_mode=True):
self._file_paths = file_paths
self._file_path_queue = []
@@ -740,6 +763,8 @@ class DagFileProcessorManager(LoggingMixin):
self._max_runs = max_runs
self._processor_factory = processor_factory
self._signal_conn = signal_conn
+ self._pickle_dags = pickle_dags
+ self._dag_ids = dag_ids
self._async_mode = async_mode
self._parallelism = conf.getint('scheduler', 'max_threads')
@@ -1216,7 +1241,8 @@ class DagFileProcessorManager(LoggingMixin):
"""
while self._parallelism - len(self._processors) > 0 and self._file_path_queue:
file_path = self._file_path_queue.pop(0)
- processor = self._processor_factory(file_path, self._zombies)
+ processor = self._processor_factory(file_path, self._zombies,
+ self._dag_ids, self._pickle_dags)
Stats.incr('dag_processing.processes')
processor.start()
@@ -1280,7 +1306,7 @@ class DagFileProcessorManager(LoggingMixin):
if not self._last_zombie_query_time or \
(now - self._last_zombie_query_time).total_seconds() > self._zombie_query_interval:
# to avoid circular imports
- from airflow.jobs import LocalTaskJob as LJ
+ from airflow.jobs.local_task_job import LocalTaskJob as LJ
self.log.info("Finding 'running' jobs without a recent heartbeat")
TI = airflow.models.TaskInstance
limit_dttm = timezone.utcnow() - timedelta(
diff --git a/airflow/utils/mixins.py b/airflow/utils/mixins.py
new file mode 100644
index 0000000..9bbc1fe
--- /dev/null
+++ b/airflow/utils/mixins.py
@@ -0,0 +1,36 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import multiprocessing
+
+from airflow.configuration import conf
+
+
+class MultiprocessingStartMethodMixin:
+ """
+ Convenience class to add support for different types of multiprocessing.
+ """
+ def _get_multiprocessing_start_method(self):
+ """
+ Determine method of creating new processes by checking if the
+ mp_start_method is set in configs, else, it uses the OS default.
+ """
+ if conf.has_option('core', 'mp_start_method'):
+ return conf.get('core', 'mp_start_method')
+
+ return multiprocessing.get_start_method()
diff --git a/tests/jobs/test_scheduler_job.py b/tests/jobs/test_scheduler_job.py
index 181d193..d636fd0 100644
--- a/tests/jobs/test_scheduler_job.py
+++ b/tests/jobs/test_scheduler_job.py
@@ -55,6 +55,7 @@ from airflow.utils.db import create_session, provide_session
from airflow.utils.state import State
from tests.compat import MagicMock, Mock, PropertyMock, mock, patch
from tests.test_core import TEST_DAG_FOLDER
+from tests.test_utils.config import conf_vars
from tests.test_utils.db import (
clear_db_dags, clear_db_errors, clear_db_pools, clear_db_runs, clear_db_sla_miss, set_default_pool_slots,
)
@@ -1377,6 +1378,31 @@ class SchedulerJobTest(unittest.TestCase):
self.assertEqual(
len(session.query(TI).filter(TI.dag_id == dag_id).all()), 0)
+ @conf_vars({("core", "mp_start_method"): "spawn"})
+ def test_scheduler_multiprocessing_with_spawn_method(self):
+ """
+ Test that the scheduler can successfully queue multiple dags in parallel
+ when using "spawn" mode of multiprocessing. (Fork is default on Linux and older OSX)
+ """
+ dag_ids = ['test_start_date_scheduling', 'test_dagrun_states_success']
+ for dag_id in dag_ids:
+ dag = self.dagbag.get_dag(dag_id)
+ dag.clear()
+
+ scheduler = SchedulerJob(dag_ids=dag_ids,
+ executor=self.null_exec,
+ subdir=os.path.join(
+ TEST_DAG_FOLDER, 'test_scheduler_dags.py'),
+ num_runs=1)
+
+ scheduler.run()
+
+ # zero tasks ran
+ dag_id = 'test_start_date_scheduling'
+ with create_session() as session:
+ self.assertEqual(
+ session.query(TaskInstance).filter(TaskInstance.dag_id == dag_id).count(), 0)
+
def test_scheduler_dagrun_once(self):
"""
Test if the scheduler does not create multiple dagruns
diff --git a/tests/test_utils/mock_executor.py b/tests/test_utils/mock_executor.py
index eb64b3b..a266682 100644
--- a/tests/test_utils/mock_executor.py
+++ b/tests/test_utils/mock_executor.py
@@ -37,10 +37,17 @@ class MockExecutor(BaseExecutor):
self.history = []
# All the tasks, in a stable sort order
self.sorted_tasks = []
- self.mock_task_results = defaultdict(lambda: State.SUCCESS)
+
+ # If multiprocessing runs in spawn mode,
+ # arguments are to be pickled but lambda is not picclable.
+ # So we should pass self.success instead of lambda.
+ self.mock_task_results = defaultdict(self.success)
super(MockExecutor, self).__init__(*args, **kwargs)
+ def success(self):
+ return State.SUCCESS
+
def heartbeat(self):
if not self.do_update:
return
diff --git a/tests/utils/test_dag_processing.py b/tests/utils/test_dag_processing.py
index 2232310..3726a8d 100644
--- a/tests/utils/test_dag_processing.py
+++ b/tests/utils/test_dag_processing.py
@@ -135,6 +135,42 @@ class settings_context(object):
sys.path.remove(self.settings_root)
+class FakeDagFileProcessorRunner(DagFileProcessor):
+ # This fake processor will return the zombies it received in constructor
+ # as its processing result w/o actually parsing anything.
+ def __init__(self, file_path, pickle_dags, dag_id_allow_list, zombies):
+ super(FakeDagFileProcessorRunner, self).__init__(file_path, pickle_dags, dag_id_allow_list, zombies)
+ self._result = zombies, 0
+
+ def start(self):
+ pass
+
+ @property
+ def start_time(self):
+ return DEFAULT_DATE
+
+ @property
+ def pid(self):
+ return 1234
+
+ @property
+ def done(self):
+ return True
+
+ @property
+ def result(self):
+ return self._result
+
+ @staticmethod
+ def _fake_dag_processor_factory(file_path, zombies, dag_ids, pickle_dags):
+ return FakeDagFileProcessorRunner(
+ file_path,
+ pickle_dags,
+ dag_ids,
+ zombies
+ )
+
+
class TestDagFileProcessorManager(unittest.TestCase):
def setUp(self):
clear_db_runs()
@@ -147,6 +183,8 @@ class TestDagFileProcessorManager(unittest.TestCase):
processor_factory=MagicMock().return_value,
processor_timeout=timedelta.max,
signal_conn=MagicMock(),
+ dag_ids=[],
+ pickle_dags=False,
async_mode=True)
mock_processor = MagicMock()
@@ -168,6 +206,8 @@ class TestDagFileProcessorManager(unittest.TestCase):
processor_factory=MagicMock().return_value,
processor_timeout=timedelta.max,
signal_conn=MagicMock(),
+ dag_ids=[],
+ pickle_dags=False,
async_mode=True)
mock_processor = MagicMock()
@@ -188,6 +228,8 @@ class TestDagFileProcessorManager(unittest.TestCase):
processor_factory=MagicMock().return_value,
processor_timeout=timedelta.max,
signal_conn=MagicMock(),
+ dag_ids=[],
+ pickle_dags=False,
async_mode=True)
dagbag = DagBag(TEST_DAG_FOLDER)
@@ -224,12 +266,14 @@ class TestDagFileProcessorManager(unittest.TestCase):
Check that the same set of zombies are passed to the dag
file processors until the next zombie detection logic is invoked.
"""
+ test_dag_path = os.path.join(TEST_DAG_FOLDER, 'test_example_bash_operator.py')
with conf_vars({('scheduler', 'max_threads'): '1',
('core', 'load_examples'): 'False'}):
- dagbag = DagBag(os.path.join(TEST_DAG_FOLDER, 'test_example_bash_operator.py'))
+ dagbag = DagBag(test_dag_path)
with create_session() as session:
session.query(LJ).delete()
dag = dagbag.get_dag('test_example_bash_operator')
+ dag.sync_to_db()
task = dag.get_task(task_id='run_this_last')
ti = TI(task, DEFAULT_DATE, State.RUNNING)
@@ -243,49 +287,14 @@ class TestDagFileProcessorManager(unittest.TestCase):
session.commit()
fake_zombies = [SimpleTaskInstance(ti)]
- class FakeDagFIleProcessor(DagFileProcessor):
- # This fake processor will return the zombies it received in constructor
- # as its processing result w/o actually parsing anything.
- def __init__(self, file_path, pickle_dags, dag_ids, zombies):
- super(FakeDagFIleProcessor, self).__init__(
- file_path, pickle_dags, dag_ids, zombies
- )
-
- self._result = zombies, 0
-
- def start(self):
- pass
-
- @property
- def start_time(self):
- return DEFAULT_DATE
-
- @property
- def pid(self):
- return 1234
-
- @property
- def done(self):
- return True
-
- @property
- def result(self):
- return self._result
-
- def processor_factory(file_path, zombies):
- return FakeDagFIleProcessor(file_path,
- False,
- [],
- zombies)
-
- test_dag_path = os.path.join(TEST_DAG_FOLDER,
- 'test_example_bash_operator.py')
async_mode = 'sqlite' not in conf.get('core', 'sql_alchemy_conn')
processor_agent = DagFileProcessorAgent(test_dag_path,
[],
1,
- processor_factory,
+ FakeDagFileProcessorRunner._fake_dag_processor_factory,
timedelta.max,
+ [],
+ False,
async_mode)
processor_agent.start()
parsing_result = []
@@ -311,6 +320,8 @@ class TestDagFileProcessorManager(unittest.TestCase):
processor_factory=MagicMock().return_value,
processor_timeout=timedelta(seconds=5),
signal_conn=MagicMock(),
+ dag_ids=[],
+ pickle_dags=False,
async_mode=True)
processor = DagFileProcessor('abc.txt', False, [], [])
@@ -330,6 +341,8 @@ class TestDagFileProcessorManager(unittest.TestCase):
processor_factory=MagicMock().return_value,
processor_timeout=timedelta(seconds=5),
signal_conn=MagicMock(),
+ dag_ids=[],
+ pickle_dags=False,
async_mode=True)
processor = DagFileProcessor('abc.txt', False, [], [])
@@ -347,8 +360,19 @@ class TestDagFileProcessorAgent(unittest.TestCase):
def tearDown(self):
# Remove any new modules imported during the test run. This lets us
# import the same source files for more than one test.
+ remove_list = []
for m in [m for m in sys.modules if m not in self.old_modules]:
- del sys.modules[m]
+ remove_list.append(m)
+
+ for mod in remove_list:
+ del sys.modules[mod]
+
+ @staticmethod
+ def _processor_factory(file_path, zombies, dag_ids, pickle_dags):
+ return DagFileProcessor(file_path,
+ pickle_dags,
+ dag_ids,
+ zombies)
def test_reload_module(self):
"""
@@ -359,16 +383,12 @@ class TestDagFileProcessorAgent(unittest.TestCase):
with settings_context(SETTINGS_FILE_VALID):
# Launch a process through DagFileProcessorAgent, which will try
# reload the logging module.
- def processor_factory(file_path, zombies):
- return DagFileProcessor(file_path,
- False,
- [],
- zombies)
test_dag_path = os.path.join(TEST_DAG_FOLDER, 'test_scheduler_dags.py')
async_mode = 'sqlite' not in conf.get('core', 'sql_alchemy_conn')
log_file_loc = conf.get('core', 'DAG_PROCESSOR_MANAGER_LOG_LOCATION')
+
try:
os.remove(log_file_loc)
except OSError:
@@ -378,8 +398,10 @@ class TestDagFileProcessorAgent(unittest.TestCase):
processor_agent = DagFileProcessorAgent(test_dag_path,
[],
0,
- processor_factory,
+ type(self)._processor_factory,
timedelta.max,
+ [],
+ False,
async_mode)
processor_agent.start()
if not async_mode:
@@ -389,22 +411,20 @@ class TestDagFileProcessorAgent(unittest.TestCase):
# Since we are reloading logging config not creating this file,
# we should expect it to be nonexistent.
+
self.assertFalse(os.path.isfile(log_file_loc))
def test_parse_once(self):
- def processor_factory(file_path, zombies):
- return DagFileProcessor(file_path,
- False,
- [],
- zombies)
test_dag_path = os.path.join(TEST_DAG_FOLDER, 'test_scheduler_dags.py')
async_mode = 'sqlite' not in conf.get('core', 'sql_alchemy_conn')
processor_agent = DagFileProcessorAgent(test_dag_path,
[test_dag_path],
1,
- processor_factory,
+ type(self)._processor_factory,
timedelta.max,
+ [],
+ False,
async_mode)
processor_agent.start()
parsing_result = []
@@ -419,11 +439,6 @@ class TestDagFileProcessorAgent(unittest.TestCase):
self.assertEqual(dag_ids.count('test_start_date_scheduling'), 1)
def test_launch_process(self):
- def processor_factory(file_path, zombies):
- return DagFileProcessor(file_path,
- False,
- [],
- zombies)
test_dag_path = os.path.join(TEST_DAG_FOLDER, 'test_scheduler_dags.py')
async_mode = 'sqlite' not in conf.get('core', 'sql_alchemy_conn')
@@ -438,8 +453,10 @@ class TestDagFileProcessorAgent(unittest.TestCase):
processor_agent = DagFileProcessorAgent(test_dag_path,
[],
0,
- processor_factory,
+ type(self)._processor_factory,
timedelta.max,
+ [],
+ False,
async_mode)
processor_agent.start()
if not async_mode: