[GitHub] [airflow] XD-DENG commented on a change in pull request #14581: Rename last_scheduler_run into last_parsed_time, and ensure it's updated in DB

[GitBox <gi...@apache.org>]

XD-DENG commented on a change in pull request #14581:
URL: https://github.com/apache/airflow/pull/14581#discussion_r588459592



##########
File path: airflow/migrations/versions/2c6edca13270_resource_based_permissions.py
##########
@@ -30,7 +30,7 @@
 
 # revision identifiers, used by Alembic.
 revision = '2c6edca13270'
-down_revision = '849da589634d'
+down_revision = '2e42bb497a22'

Review comment:
       If we don't adjust the order, at migration step `Running upgrade 849da589634d -> 2c6edca13270, Resource based permissions.`, we will get error below. If there is better/more proper way to handle this, please let me know (I'm improving my understanding on Alembic :) )
   
   ```
   Traceback (most recent call last):
     File "/Users/XD/test_airflow/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1277, in _execute_context
       cursor, statement, parameters, context
     File "/Users/XD/test_airflow/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 608, in do_execute
       cursor.execute(statement, parameters)
   sqlite3.OperationalError: no such column: dag.last_scheduler_run
   
   The above exception was the direct cause of the following exception:
   
   Traceback (most recent call last):
     File "/Users/XD/test_airflow/bin/airflow", line 8, in <module>
       sys.exit(main())
     File "/Users/XD/test_airflow/lib/python3.7/site-packages/airflow/__main__.py", line 40, in main
       args.func(args)
     File "/Users/XD/test_airflow/lib/python3.7/site-packages/airflow/cli/cli_parser.py", line 48, in command
       return func(*args, **kwargs)
     File "/Users/XD/test_airflow/lib/python3.7/site-packages/airflow/cli/commands/db_command.py", line 39, in resetdb
       db.resetdb()
     File "/Users/XD/test_airflow/lib/python3.7/site-packages/airflow/utils/db.py", line 697, in resetdb
       initdb()
     File "/Users/XD/test_airflow/lib/python3.7/site-packages/airflow/utils/db.py", line 549, in initdb
       upgradedb()
     File "/Users/XD/test_airflow/lib/python3.7/site-packages/airflow/utils/db.py", line 684, in upgradedb
       command.upgrade(config, 'heads')
     File "/Users/XD/test_airflow/lib/python3.7/site-packages/alembic/command.py", line 294, in upgrade
       script.run_env()
     File "/Users/XD/test_airflow/lib/python3.7/site-packages/alembic/script/base.py", line 481, in run_env
       util.load_python_file(self.dir, "env.py")
     File "/Users/XD/test_airflow/lib/python3.7/site-packages/alembic/util/pyfiles.py", line 97, in load_python_file
       module = load_module_py(module_id, path)
     File "/Users/XD/test_airflow/lib/python3.7/site-packages/alembic/util/compat.py", line 182, in load_module_py
       spec.loader.exec_module(module)
     File "<frozen importlib._bootstrap_external>", line 728, in exec_module
     File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
     File "/Users/XD/test_airflow/lib/python3.7/site-packages/airflow/migrations/env.py", line 108, in <module>
       run_migrations_online()
     File "/Users/XD/test_airflow/lib/python3.7/site-packages/airflow/migrations/env.py", line 102, in run_migrations_online
       context.run_migrations()
     File "<string>", line 8, in run_migrations
     File "/Users/XD/test_airflow/lib/python3.7/site-packages/alembic/runtime/environment.py", line 813, in run_migrations
       self.get_context().run_migrations(**kw)
     File "/Users/XD/test_airflow/lib/python3.7/site-packages/alembic/runtime/migration.py", line 560, in run_migrations
       step.migration_fn(**kw)
     File "/Users/XD/test_airflow/lib/python3.7/site-packages/airflow/migrations/versions/2c6edca13270_resource_based_permissions.py", line 314, in upgrade
       remap_permissions()
     File "/Users/XD/test_airflow/lib/python3.7/site-packages/airflow/migrations/versions/2c6edca13270_resource_based_permissions.py", line 289, in remap_permissions
       appbuilder = create_app(config={'FAB_UPDATE_PERMS': False}).appbuilder
     File "/Users/XD/test_airflow/lib/python3.7/site-packages/airflow/www/app.py", line 123, in create_app
       sync_appbuilder_roles(flask_app)
     File "/Users/XD/test_airflow/lib/python3.7/site-packages/airflow/www/app.py", line 64, in sync_appbuilder_roles
       security_manager.sync_roles()
     File "/Users/XD/test_airflow/lib/python3.7/site-packages/airflow/www/security.py", line 526, in sync_roles
       self.create_dag_specific_permissions()
     File "/Users/XD/test_airflow/lib/python3.7/site-packages/airflow/utils/session.py", line 65, in wrapper
       return func(*args, session=session, **kwargs)
     File "/Users/XD/test_airflow/lib/python3.7/site-packages/airflow/www/security.py", line 479, in create_dag_specific_permissions
       .filter(or_(models.DagModel.is_active, models.DagModel.is_paused))
     File "/Users/XD/test_airflow/lib/python3.7/site-packages/sqlalchemy/orm/query.py", line 3373, in all
       return list(self)
     File "/Users/XD/test_airflow/lib/python3.7/site-packages/sqlalchemy/orm/query.py", line 3535, in __iter__
       return self._execute_and_instances(context)
     File "/Users/XD/test_airflow/lib/python3.7/site-packages/sqlalchemy/orm/query.py", line 3560, in _execute_and_instances
       result = conn.execute(querycontext.statement, self._params)
     File "/Users/XD/test_airflow/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1011, in execute
       return meth(self, multiparams, params)
     File "/Users/XD/test_airflow/lib/python3.7/site-packages/sqlalchemy/sql/elements.py", line 298, in _execute_on_connection
       return connection._execute_clauseelement(self, multiparams, params)
     File "/Users/XD/test_airflow/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1130, in _execute_clauseelement
       distilled_params,
     File "/Users/XD/test_airflow/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1317, in _execute_context
       e, statement, parameters, cursor, context
     File "/Users/XD/test_airflow/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1511, in _handle_dbapi_exception
       sqlalchemy_exception, with_traceback=exc_info[2], from_=e
     File "/Users/XD/test_airflow/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 182, in raise_
       raise exception
     File "/Users/XD/test_airflow/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1277, in _execute_context
       cursor, statement, parameters, context
     File "/Users/XD/test_airflow/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 608, in do_execute
       cursor.execute(statement, parameters)
   sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) no such column: dag.last_scheduler_run
   [SQL: SELECT dag.dag_id AS dag_dag_id, dag.root_dag_id AS dag_root_dag_id, dag.is_paused AS dag_is_paused, dag.is_subdag AS dag_is_subdag, dag.is_active AS dag_is_active, dag.last_scheduler_run AS dag_last_scheduler_run, dag.last_pickled AS dag_last_pickled, dag.last_expired AS dag_last_expired, dag.scheduler_lock AS dag_scheduler_lock, dag.pickle_id AS dag_pickle_id, dag.fileloc AS dag_fileloc, dag.owners AS dag_owners, dag.description AS dag_description, dag.default_view AS dag_default_view, dag.schedule_interval AS dag_schedule_interval, dag.concurrency AS dag_concurrency, dag.has_task_concurrency_limits AS dag_has_task_concurrency_limits, dag.next_dagrun AS dag_next_dagrun, dag.next_dagrun_create_after AS dag_next_dagrun_create_after
   FROM dag
   WHERE dag.is_active = 1 OR dag.is_paused = 1]
   (Background on this error at: http://sqlalche.me/e/13/e3q8)
   ``` 




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org