You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by GitBox <gi...@apache.org> on 2021/02/25 15:41:26 UTC
[GitHub] [airflow] turbaszek commented on a change in pull request #14420: Don't create unittest.cfg when not running in unit test mode
turbaszek commented on a change in pull request #14420:
URL: https://github.com/apache/airflow/pull/14420#discussion_r582939080
##########
File path: airflow/configuration.py
##########
@@ -779,65 +758,93 @@ def get_airflow_test_config(airflow_home):
return expand_env_var(os.environ['AIRFLOW_TEST_CONFIG'])
-TEST_CONFIG_FILE = get_airflow_test_config(AIRFLOW_HOME)
+def _generate_fernet_key():
+ from cryptography.fernet import Fernet
-# only generate a Fernet key if we need to create a new config file
-if not os.path.isfile(TEST_CONFIG_FILE) or not os.path.isfile(AIRFLOW_CONFIG):
- FERNET_KEY = Fernet.generate_key().decode()
-else:
- FERNET_KEY = ''
+ return Fernet.generate_key().decode()
-SECRET_KEY = b64encode(os.urandom(16)).decode('utf-8')
-TEMPLATE_START = '# ----------------------- TEMPLATE BEGINS HERE -----------------------'
-if not os.path.isfile(TEST_CONFIG_FILE):
- log.info('Creating new Airflow config file for unit tests in: %s', TEST_CONFIG_FILE)
- with open(TEST_CONFIG_FILE, 'w') as file:
- cfg = parameterized_config(TEST_CONFIG)
- file.write(cfg.split(TEMPLATE_START)[-1].strip())
-if not os.path.isfile(AIRFLOW_CONFIG):
- log.info('Creating new Airflow config file in: %s', AIRFLOW_CONFIG)
- with open(AIRFLOW_CONFIG, 'w') as file:
- cfg = parameterized_config(DEFAULT_CONFIG)
- cfg = cfg.split(TEMPLATE_START)[-1].strip()
- file.write(cfg)
-
-log.info("Reading the config from %s", AIRFLOW_CONFIG)
-
-conf = AirflowConfigParser(default_config=parameterized_config(DEFAULT_CONFIG))
-
-conf.read(AIRFLOW_CONFIG)
-
-if conf.has_option('core', 'AIRFLOW_HOME'):
- msg = (
- 'Specifying both AIRFLOW_HOME environment variable and airflow_home '
- 'in the config file is deprecated. Please use only the AIRFLOW_HOME '
- 'environment variable and remove the config file entry.'
- )
- if 'AIRFLOW_HOME' in os.environ:
- warnings.warn(msg, category=DeprecationWarning)
- elif conf.get('core', 'airflow_home') == AIRFLOW_HOME:
- warnings.warn(
- 'Specifying airflow_home in the config file is deprecated. As you '
- 'have left it at the default value you should remove the setting '
- 'from your airflow.cfg and suffer no change in behaviour.',
- category=DeprecationWarning,
- )
+def initialize_config():
+ """
+ Load the Airflow config files.
+
+ Called for you automatically as part of the Airflow boot process.
+ """
+ global FERNET_KEY, AIRFLOW_HOME
+
+ default_config = _parameterized_config_from_template('default_airflow.cfg')
+
+ conf = AirflowConfigParser(default_config=default_config)
+
+ if conf.getboolean('core', 'unit_test_mode'):
+ # Load test config only
+ if not os.path.isfile(TEST_CONFIG_FILE):
+ from cryptography.fernet import Fernet
+
+ log.info('Creating new Airflow config file for unit tests in: %s', TEST_CONFIG_FILE)
+ pathlib.Path(AIRFLOW_HOME).mkdir(parents=True, exist_ok=True)
+
+ FERNET_KEY = Fernet.generate_key().decode()
+
+ with open(TEST_CONFIG_FILE, 'w') as file:
+ cfg = _parameterized_config_from_template('default_test.cfg')
+ file.write(cfg)
+
+ conf.load_test_config()
else:
- AIRFLOW_HOME = conf.get('core', 'airflow_home')
- warnings.warn(msg, category=DeprecationWarning)
+ # Load normal config
+ if not os.path.isfile(AIRFLOW_CONFIG):
+ from cryptography.fernet import Fernet
+ log.info('Creating new Airflow config file in: %s', AIRFLOW_CONFIG)
+ pathlib.Path(AIRFLOW_HOME).mkdir(parents=True, exist_ok=True)
-WEBSERVER_CONFIG = AIRFLOW_HOME + '/webserver_config.py'
+ FERNET_KEY = Fernet.generate_key().decode()
-if not os.path.isfile(WEBSERVER_CONFIG):
- log.info('Creating new FAB webserver config file in: %s', WEBSERVER_CONFIG)
- DEFAULT_WEBSERVER_CONFIG, _ = _read_default_config_file('default_webserver_config.py')
- with open(WEBSERVER_CONFIG, 'w') as file:
- file.write(DEFAULT_WEBSERVER_CONFIG)
+ with open(AIRFLOW_CONFIG, 'w') as file:
+ file.write(default_config)
-if conf.getboolean('core', 'unit_test_mode'):
- conf.load_test_config()
+ log.info("Reading the config from %s", AIRFLOW_CONFIG)
+
+ conf.read(AIRFLOW_CONFIG)
+
+ if conf.has_option('core', 'AIRFLOW_HOME'):
+ msg = (
+ 'Specifying both AIRFLOW_HOME environment variable and airflow_home '
+ 'in the config file is deprecated. Please use only the AIRFLOW_HOME '
+ 'environment variable and remove the config file entry.'
+ )
+ if 'AIRFLOW_HOME' in os.environ:
+ warnings.warn(msg, category=DeprecationWarning)
+ elif conf.get('core', 'airflow_home') == AIRFLOW_HOME:
+ warnings.warn(
+ 'Specifying airflow_home in the config file is deprecated. As you '
+ 'have left it at the default value you should remove the setting '
+ 'from your airflow.cfg and suffer no change in behaviour.',
+ category=DeprecationWarning,
+ )
+ else:
+ AIRFLOW_HOME = conf.get('core', 'airflow_home')
+ warnings.warn(msg, category=DeprecationWarning)
+
+ # They _might_ have set unit_test_mode in the airflow.cfg, we still
+ # want to respect that and then load the unittests.cfg
+ if conf.getboolean('core', 'unit_test_mode'):
+ conf.load_test_config()
+
+ # Make it no longer a proxy variable, just set it to an actual string
+ global WEBSERVER_CONFIG
+ WEBSERVER_CONFIG = AIRFLOW_HOME + '/webserver_config.py'
Review comment:
```suggestion
WEBSERVER_CONFIG = os.path.join(AIRFLOW_HOME, 'webserver_config.py')
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org