You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by sa...@apache.org on 2017/06/09 00:27:24 UTC
incubator-airflow git commit: [AIRFLOW-1289] Removes restriction on
number of scheduler threads
Repository: incubator-airflow
Updated Branches:
refs/heads/master 7af20fe45 -> 8e23d2870
[AIRFLOW-1289] Removes restriction on number of scheduler threads
This removes the restriction that the number of
threads can be at most
the number of CPU cores. There's no reason to have
this restriction.
Closes #2353 from saguziel/aguziel-increase-cores
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/8e23d287
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/8e23d287
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/8e23d287
Branch: refs/heads/master
Commit: 8e23d28705e1f0d9c4ab76bb2318025bab3aef0a
Parents: 7af20fe
Author: Alex Guziel <al...@airbnb.com>
Authored: Thu Jun 8 17:27:19 2017 -0700
Committer: Alex Guziel <al...@airbnb.com>
Committed: Thu Jun 8 17:27:19 2017 -0700
----------------------------------------------------------------------
airflow/config_templates/default_airflow.cfg | 3 +--
airflow/jobs.py | 2 +-
2 files changed, 2 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/8e23d287/airflow/config_templates/default_airflow.cfg
----------------------------------------------------------------------
diff --git a/airflow/config_templates/default_airflow.cfg b/airflow/config_templates/default_airflow.cfg
index 2b5fb5d..c6c1da2 100644
--- a/airflow/config_templates/default_airflow.cfg
+++ b/airflow/config_templates/default_airflow.cfg
@@ -326,8 +326,7 @@ statsd_port = 8125
statsd_prefix = airflow
# The scheduler can run multiple threads in parallel to schedule dags.
-# This defines how many threads will run. However airflow will never
-# use more threads than the amount of cpu cores available.
+# This defines how many threads will run.
max_threads = 2
authenticate = False
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/8e23d287/airflow/jobs.py
----------------------------------------------------------------------
diff --git a/airflow/jobs.py b/airflow/jobs.py
index 508564a..57688d3 100644
--- a/airflow/jobs.py
+++ b/airflow/jobs.py
@@ -528,7 +528,7 @@ class SchedulerJob(BaseJob):
super(SchedulerJob, self).__init__(*args, **kwargs)
self.heartrate = conf.getint('scheduler', 'SCHEDULER_HEARTBEAT_SEC')
- self.max_threads = min(conf.getint('scheduler', 'max_threads'), multiprocessing.cpu_count())
+ self.max_threads = conf.getint('scheduler', 'max_threads')
self.using_sqlite = False
if 'sqlite' in conf.get('core', 'sql_alchemy_conn'):
if self.max_threads > 1: