You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@airflow.apache.org by Kyle Hamlin <ha...@gmail.com> on 2018/04/16 19:09:12 UTC
Trouble with remote s3 logging
This morning I tried to upgrade to the newer version of the logging config
file but I keep getting the following a TypeError for my database session.
I know my credentials are correct so I'm confused why this is happening now.
Has anyone experiences this? Note that I'm installing Airflow from master.
*Config*
AIRFLOW__CORE__LOGGING_LEVEL=WARN
AIRFLOW__CORE__REMOTE_LOGGING=True
AIRFLOW__CORE__REMOTE_LOG_CONN_ID=s3_logger
AIRFLOW__CORE__REMOTE_BASE_LOG_FOLDER=s3://airflow/logs
AIRFLOW__CORE__LOGGING_CONFIG_CLASS=config.log_config.DEFAULT_LOGGING_CONFIG
AIRFLOW__SCHEDULER__CHILD_PROCESS_LOG_DIRECTORY=s3://airflow/logs
*Session NoneType error*
Traceback (most recent call last):
File
"/usr/local/lib/python3.6/site-packages/airflow/utils/log/s3_task_handler.py",
line 171, in s3_write
encrypt=configuration.conf.getboolean('core', 'ENCRYPT_S3_LOGS'),
File "/usr/local/lib/python3.6/site-packages/airflow/hooks/S3_hook.py",
line 274, in load_string
encrypt=encrypt)
File "/usr/local/lib/python3.6/site-packages/airflow/hooks/S3_hook.py",
line 313, in load_bytes
client = self.get_conn()
File "/usr/local/lib/python3.6/site-packages/airflow/hooks/S3_hook.py",
line 34, in get_conn
return self.get_client_type('s3')
File
"/usr/local/lib/python3.6/site-packages/airflow/contrib/hooks/aws_hook.py",
line 151, in get_client_type
session, endpoint_url = self._get_credentials(region_name)
File
"/usr/local/lib/python3.6/site-packages/airflow/contrib/hooks/aws_hook.py",
line 97, in _get_credentials
connection_object = self.get_connection(self.aws_conn_id)
File
"/usr/local/lib/python3.6/site-packages/airflow/hooks/base_hook.py", line
82, in get_connection
conn = random.choice(cls.get_connections(conn_id))
File
"/usr/local/lib/python3.6/site-packages/airflow/hooks/base_hook.py", line
77, in get_connections
conns = cls._get_connections_from_db(conn_id)
File "/usr/local/lib/python3.6/site-packages/airflow/utils/db.py", line
72, in wrapper
with create_session() as session:
File "/usr/local/lib/python3.6/contextlib.py", line 81, in __enter__
return next(self.gen)
File "/usr/local/lib/python3.6/site-packages/airflow/utils/db.py", line
41, in create_session
session = settings.Session()
TypeError: 'NoneType' object is not callable
*TypeError must be str not tuple*
[2018-04-16 18:37:28,200] ERROR in app: Exception on
/admin/airflow/get_logs_with_metadata [GET]
Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/flask/app.py", line 1982,
in wsgi_app
response = self.full_dispatch_request()
File "/usr/local/lib/python3.6/site-packages/flask/app.py", line 1614,
in full_dispatch_request
rv = self.handle_user_exception(e)
File "/usr/local/lib/python3.6/site-packages/flask/app.py", line 1517,
in handle_user_exception
reraise(exc_type, exc_value, tb)
File "/usr/local/lib/python3.6/site-packages/flask/_compat.py", line 33,
in reraise
raise value
File "/usr/local/lib/python3.6/site-packages/flask/app.py", line 1612,
in full_dispatch_request
rv = self.dispatch_request()
File "/usr/local/lib/python3.6/site-packages/flask/app.py", line 1598,
in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "/usr/local/lib/python3.6/site-packages/flask_admin/base.py", line
69, in inner
return self._run_view(f, *args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/flask_admin/base.py", line
368, in _run_view
return fn(self, *args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/flask_login.py", line 755,
in decorated_view
return func(*args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/airflow/www/utils.py", line
269, in wrapper
return f(*args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/airflow/utils/db.py", line
74, in wrapper
return func(*args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/airflow/www/views.py", line
770, in get_logs_with_metadata
logs, metadatas = handler.read(ti, try_number, metadata=metadata)
File
"/usr/local/lib/python3.6/site-packages/airflow/utils/log/file_task_handler.py",
line 165, in read
logs[i] += log
TypeError: must be str, not tuple
Re: Trouble with remote s3 logging
Posted by Kyle Hamlin <ha...@gmail.com>.
Yeah here is the issue: https://issues.apache.org/jira/browse/AIRFLOW-2363
On Wed, Apr 18, 2018 at 2:17 AM yrqls21@gmail.com <yr...@gmail.com> wrote:
>
>
> On 2018/04/16 19:09:12, Kyle Hamlin <ha...@gmail.com> wrote:
> > This morning I tried to upgrade to the newer version of the logging
> config
> > file but I keep getting the following a TypeError for my database
> session.
> > I know my credentials are correct so I'm confused why this is happening
> now.
> >
> > Has anyone experiences this? Note that I'm installing Airflow from
> master.
> >
> > *Config*
> > AIRFLOW__CORE__LOGGING_LEVEL=WARN
> > AIRFLOW__CORE__REMOTE_LOGGING=True
> > AIRFLOW__CORE__REMOTE_LOG_CONN_ID=s3_logger
> > AIRFLOW__CORE__REMOTE_BASE_LOG_FOLDER=s3://airflow/logs
> >
> AIRFLOW__CORE__LOGGING_CONFIG_CLASS=config.log_config.DEFAULT_LOGGING_CONFIG
> > AIRFLOW__SCHEDULER__CHILD_PROCESS_LOG_DIRECTORY=s3://airflow/logs
> >
> > *Session NoneType error*
> > Traceback (most recent call last):
> > File
> >
> "/usr/local/lib/python3.6/site-packages/airflow/utils/log/s3_task_handler.py",
> > line 171, in s3_write
> > encrypt=configuration.conf.getboolean('core', 'ENCRYPT_S3_LOGS'),
> > File
> "/usr/local/lib/python3.6/site-packages/airflow/hooks/S3_hook.py",
> > line 274, in load_string
> > encrypt=encrypt)
> > File
> "/usr/local/lib/python3.6/site-packages/airflow/hooks/S3_hook.py",
> > line 313, in load_bytes
> > client = self.get_conn()
> > File
> "/usr/local/lib/python3.6/site-packages/airflow/hooks/S3_hook.py",
> > line 34, in get_conn
> > return self.get_client_type('s3')
> > File
> >
> "/usr/local/lib/python3.6/site-packages/airflow/contrib/hooks/aws_hook.py",
> > line 151, in get_client_type
> > session, endpoint_url = self._get_credentials(region_name)
> > File
> >
> "/usr/local/lib/python3.6/site-packages/airflow/contrib/hooks/aws_hook.py",
> > line 97, in _get_credentials
> > connection_object = self.get_connection(self.aws_conn_id)
> > File
> > "/usr/local/lib/python3.6/site-packages/airflow/hooks/base_hook.py", line
> > 82, in get_connection
> > conn = random.choice(cls.get_connections(conn_id))
> > File
> > "/usr/local/lib/python3.6/site-packages/airflow/hooks/base_hook.py", line
> > 77, in get_connections
> > conns = cls._get_connections_from_db(conn_id)
> > File "/usr/local/lib/python3.6/site-packages/airflow/utils/db.py",
> line
> > 72, in wrapper
> > with create_session() as session:
> > File "/usr/local/lib/python3.6/contextlib.py", line 81, in __enter__
> > return next(self.gen)
> > File "/usr/local/lib/python3.6/site-packages/airflow/utils/db.py",
> line
> > 41, in create_session
> > session = settings.Session()
> > TypeError: 'NoneType' object is not callable
> >
> > *TypeError must be str not tuple*
> > [2018-04-16 18:37:28,200] ERROR in app: Exception on
> > /admin/airflow/get_logs_with_metadata [GET]
> > Traceback (most recent call last):
> > File "/usr/local/lib/python3.6/site-packages/flask/app.py", line 1982,
> > in wsgi_app
> > response = self.full_dispatch_request()
> > File "/usr/local/lib/python3.6/site-packages/flask/app.py", line 1614,
> > in full_dispatch_request
> > rv = self.handle_user_exception(e)
> > File "/usr/local/lib/python3.6/site-packages/flask/app.py", line 1517,
> > in handle_user_exception
> > reraise(exc_type, exc_value, tb)
> > File "/usr/local/lib/python3.6/site-packages/flask/_compat.py", line
> 33,
> > in reraise
> > raise value
> > File "/usr/local/lib/python3.6/site-packages/flask/app.py", line 1612,
> > in full_dispatch_request
> > rv = self.dispatch_request()
> > File "/usr/local/lib/python3.6/site-packages/flask/app.py", line 1598,
> > in dispatch_request
> > return self.view_functions[rule.endpoint](**req.view_args)
> > File "/usr/local/lib/python3.6/site-packages/flask_admin/base.py",
> line
> > 69, in inner
> > return self._run_view(f, *args, **kwargs)
> > File "/usr/local/lib/python3.6/site-packages/flask_admin/base.py",
> line
> > 368, in _run_view
> > return fn(self, *args, **kwargs)
> > File "/usr/local/lib/python3.6/site-packages/flask_login.py", line
> 755,
> > in decorated_view
> > return func(*args, **kwargs)
> > File "/usr/local/lib/python3.6/site-packages/airflow/www/utils.py",
> line
> > 269, in wrapper
> > return f(*args, **kwargs)
> > File "/usr/local/lib/python3.6/site-packages/airflow/utils/db.py",
> line
> > 74, in wrapper
> > return func(*args, **kwargs)
> > File "/usr/local/lib/python3.6/site-packages/airflow/www/views.py",
> line
> > 770, in get_logs_with_metadata
> > logs, metadatas = handler.read(ti, try_number, metadata=metadata)
> > File
> >
> "/usr/local/lib/python3.6/site-packages/airflow/utils/log/file_task_handler.py",
> > line 165, in read
> > logs[i] += log
> > TypeError: must be str, not tuple
> > I'm fairly familar with that part of code, would you ping me in the
> issue/PR you created?
>
Re: Trouble with remote s3 logging
Posted by .
On 2018/04/16 19:09:12, Kyle Hamlin <ha...@gmail.com> wrote:
> This morning I tried to upgrade to the newer version of the logging config
> file but I keep getting the following a TypeError for my database session.
> I know my credentials are correct so I'm confused why this is happening now.
>
> Has anyone experiences this? Note that I'm installing Airflow from master.
>
> *Config*
> AIRFLOW__CORE__LOGGING_LEVEL=WARN
> AIRFLOW__CORE__REMOTE_LOGGING=True
> AIRFLOW__CORE__REMOTE_LOG_CONN_ID=s3_logger
> AIRFLOW__CORE__REMOTE_BASE_LOG_FOLDER=s3://airflow/logs
> AIRFLOW__CORE__LOGGING_CONFIG_CLASS=config.log_config.DEFAULT_LOGGING_CONFIG
> AIRFLOW__SCHEDULER__CHILD_PROCESS_LOG_DIRECTORY=s3://airflow/logs
>
> *Session NoneType error*
> Traceback (most recent call last):
> File
> "/usr/local/lib/python3.6/site-packages/airflow/utils/log/s3_task_handler.py",
> line 171, in s3_write
> encrypt=configuration.conf.getboolean('core', 'ENCRYPT_S3_LOGS'),
> File "/usr/local/lib/python3.6/site-packages/airflow/hooks/S3_hook.py",
> line 274, in load_string
> encrypt=encrypt)
> File "/usr/local/lib/python3.6/site-packages/airflow/hooks/S3_hook.py",
> line 313, in load_bytes
> client = self.get_conn()
> File "/usr/local/lib/python3.6/site-packages/airflow/hooks/S3_hook.py",
> line 34, in get_conn
> return self.get_client_type('s3')
> File
> "/usr/local/lib/python3.6/site-packages/airflow/contrib/hooks/aws_hook.py",
> line 151, in get_client_type
> session, endpoint_url = self._get_credentials(region_name)
> File
> "/usr/local/lib/python3.6/site-packages/airflow/contrib/hooks/aws_hook.py",
> line 97, in _get_credentials
> connection_object = self.get_connection(self.aws_conn_id)
> File
> "/usr/local/lib/python3.6/site-packages/airflow/hooks/base_hook.py", line
> 82, in get_connection
> conn = random.choice(cls.get_connections(conn_id))
> File
> "/usr/local/lib/python3.6/site-packages/airflow/hooks/base_hook.py", line
> 77, in get_connections
> conns = cls._get_connections_from_db(conn_id)
> File "/usr/local/lib/python3.6/site-packages/airflow/utils/db.py", line
> 72, in wrapper
> with create_session() as session:
> File "/usr/local/lib/python3.6/contextlib.py", line 81, in __enter__
> return next(self.gen)
> File "/usr/local/lib/python3.6/site-packages/airflow/utils/db.py", line
> 41, in create_session
> session = settings.Session()
> TypeError: 'NoneType' object is not callable
>
> *TypeError must be str not tuple*
> [2018-04-16 18:37:28,200] ERROR in app: Exception on
> /admin/airflow/get_logs_with_metadata [GET]
> Traceback (most recent call last):
> File "/usr/local/lib/python3.6/site-packages/flask/app.py", line 1982,
> in wsgi_app
> response = self.full_dispatch_request()
> File "/usr/local/lib/python3.6/site-packages/flask/app.py", line 1614,
> in full_dispatch_request
> rv = self.handle_user_exception(e)
> File "/usr/local/lib/python3.6/site-packages/flask/app.py", line 1517,
> in handle_user_exception
> reraise(exc_type, exc_value, tb)
> File "/usr/local/lib/python3.6/site-packages/flask/_compat.py", line 33,
> in reraise
> raise value
> File "/usr/local/lib/python3.6/site-packages/flask/app.py", line 1612,
> in full_dispatch_request
> rv = self.dispatch_request()
> File "/usr/local/lib/python3.6/site-packages/flask/app.py", line 1598,
> in dispatch_request
> return self.view_functions[rule.endpoint](**req.view_args)
> File "/usr/local/lib/python3.6/site-packages/flask_admin/base.py", line
> 69, in inner
> return self._run_view(f, *args, **kwargs)
> File "/usr/local/lib/python3.6/site-packages/flask_admin/base.py", line
> 368, in _run_view
> return fn(self, *args, **kwargs)
> File "/usr/local/lib/python3.6/site-packages/flask_login.py", line 755,
> in decorated_view
> return func(*args, **kwargs)
> File "/usr/local/lib/python3.6/site-packages/airflow/www/utils.py", line
> 269, in wrapper
> return f(*args, **kwargs)
> File "/usr/local/lib/python3.6/site-packages/airflow/utils/db.py", line
> 74, in wrapper
> return func(*args, **kwargs)
> File "/usr/local/lib/python3.6/site-packages/airflow/www/views.py", line
> 770, in get_logs_with_metadata
> logs, metadatas = handler.read(ti, try_number, metadata=metadata)
> File
> "/usr/local/lib/python3.6/site-packages/airflow/utils/log/file_task_handler.py",
> line 165, in read
> logs[i] += log
> TypeError: must be str, not tuple
> I'm fairly familar with that part of code, would you ping me in the issue/PR you created?
Re: Trouble with remote s3 logging
Posted by "Driesprong, Fokko" <fo...@driesprong.frl>.
Hi Kyle,
Thanks for reaching out. This looks like a bug. Could you open a Jira issue
describing this issue?
This seems to be something that should be rather easy fo fix. Since it
tries to append a tuple instead of a string. I would expect that more
people would encounter this bug.
Cheers, Fokko
2018-04-16 21:09 GMT+02:00 Kyle Hamlin <ha...@gmail.com>:
> This morning I tried to upgrade to the newer version of the logging config
> file but I keep getting the following a TypeError for my database session.
> I know my credentials are correct so I'm confused why this is happening
> now.
>
> Has anyone experiences this? Note that I'm installing Airflow from master.
>
> *Config*
> AIRFLOW__CORE__LOGGING_LEVEL=WARN
> AIRFLOW__CORE__REMOTE_LOGGING=True
> AIRFLOW__CORE__REMOTE_LOG_CONN_ID=s3_logger
> AIRFLOW__CORE__REMOTE_BASE_LOG_FOLDER=s3://airflow/logs
> AIRFLOW__CORE__LOGGING_CONFIG_CLASS=config.log_config.
> DEFAULT_LOGGING_CONFIG
> AIRFLOW__SCHEDULER__CHILD_PROCESS_LOG_DIRECTORY=s3://airflow/logs
>
> *Session NoneType error*
> Traceback (most recent call last):
> File
> "/usr/local/lib/python3.6/site-packages/airflow/utils/
> log/s3_task_handler.py",
> line 171, in s3_write
> encrypt=configuration.conf.getboolean('core', 'ENCRYPT_S3_LOGS'),
> File "/usr/local/lib/python3.6/site-packages/airflow/hooks/S3_hook.py",
> line 274, in load_string
> encrypt=encrypt)
> File "/usr/local/lib/python3.6/site-packages/airflow/hooks/S3_hook.py",
> line 313, in load_bytes
> client = self.get_conn()
> File "/usr/local/lib/python3.6/site-packages/airflow/hooks/S3_hook.py",
> line 34, in get_conn
> return self.get_client_type('s3')
> File
> "/usr/local/lib/python3.6/site-packages/airflow/contrib/
> hooks/aws_hook.py",
> line 151, in get_client_type
> session, endpoint_url = self._get_credentials(region_name)
> File
> "/usr/local/lib/python3.6/site-packages/airflow/contrib/
> hooks/aws_hook.py",
> line 97, in _get_credentials
> connection_object = self.get_connection(self.aws_conn_id)
> File
> "/usr/local/lib/python3.6/site-packages/airflow/hooks/base_hook.py", line
> 82, in get_connection
> conn = random.choice(cls.get_connections(conn_id))
> File
> "/usr/local/lib/python3.6/site-packages/airflow/hooks/base_hook.py", line
> 77, in get_connections
> conns = cls._get_connections_from_db(conn_id)
> File "/usr/local/lib/python3.6/site-packages/airflow/utils/db.py", line
> 72, in wrapper
> with create_session() as session:
> File "/usr/local/lib/python3.6/contextlib.py", line 81, in __enter__
> return next(self.gen)
> File "/usr/local/lib/python3.6/site-packages/airflow/utils/db.py", line
> 41, in create_session
> session = settings.Session()
> TypeError: 'NoneType' object is not callable
>
> *TypeError must be str not tuple*
> [2018-04-16 18:37:28,200] ERROR in app: Exception on
> /admin/airflow/get_logs_with_metadata [GET]
> Traceback (most recent call last):
> File "/usr/local/lib/python3.6/site-packages/flask/app.py", line 1982,
> in wsgi_app
> response = self.full_dispatch_request()
> File "/usr/local/lib/python3.6/site-packages/flask/app.py", line 1614,
> in full_dispatch_request
> rv = self.handle_user_exception(e)
> File "/usr/local/lib/python3.6/site-packages/flask/app.py", line 1517,
> in handle_user_exception
> reraise(exc_type, exc_value, tb)
> File "/usr/local/lib/python3.6/site-packages/flask/_compat.py", line
> 33,
> in reraise
> raise value
> File "/usr/local/lib/python3.6/site-packages/flask/app.py", line 1612,
> in full_dispatch_request
> rv = self.dispatch_request()
> File "/usr/local/lib/python3.6/site-packages/flask/app.py", line 1598,
> in dispatch_request
> return self.view_functions[rule.endpoint](**req.view_args)
> File "/usr/local/lib/python3.6/site-packages/flask_admin/base.py", line
> 69, in inner
> return self._run_view(f, *args, **kwargs)
> File "/usr/local/lib/python3.6/site-packages/flask_admin/base.py", line
> 368, in _run_view
> return fn(self, *args, **kwargs)
> File "/usr/local/lib/python3.6/site-packages/flask_login.py", line 755,
> in decorated_view
> return func(*args, **kwargs)
> File "/usr/local/lib/python3.6/site-packages/airflow/www/utils.py",
> line
> 269, in wrapper
> return f(*args, **kwargs)
> File "/usr/local/lib/python3.6/site-packages/airflow/utils/db.py", line
> 74, in wrapper
> return func(*args, **kwargs)
> File "/usr/local/lib/python3.6/site-packages/airflow/www/views.py",
> line
> 770, in get_logs_with_metadata
> logs, metadatas = handler.read(ti, try_number, metadata=metadata)
> File
> "/usr/local/lib/python3.6/site-packages/airflow/utils/
> log/file_task_handler.py",
> line 165, in read
> logs[i] += log
> TypeError: must be str, not tuple
>