[1/7] incubator-airflow-site git commit: New version of the docs

[ma...@apache.org]

Repository: incubator-airflow-site
Updated Branches:
  refs/heads/asf-site 457b698d9 -> 4af0850c3


http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/tutorial.html
----------------------------------------------------------------------
diff --git a/tutorial.html b/tutorial.html
index abcbbc0..17148bb 100644
--- a/tutorial.html
+++ b/tutorial.html
@@ -190,7 +190,7 @@ complicated, a line by line explanation follows below.</p>
 
 <span class="n">dag</span> <span class="o">=</span> <span class="n">DAG</span><span class="p">(</span><span class="s1">&#39;tutorial&#39;</span><span class="p">,</span> <span class="n">default_args</span><span class="o">=</span><span class="n">default_args</span><span class="p">)</span>
 
-<span class="c1"># t1, t2 and t3 are examples of tasks created by instatiating operators</span>
+<span class="c1"># t1, t2 and t3 are examples of tasks created by instantiating operators</span>
 <span class="n">t1</span> <span class="o">=</span> <span class="n">BashOperator</span><span class="p">(</span>
     <span class="n">task_id</span><span class="o">=</span><span class="s1">&#39;print_date&#39;</span><span class="p">,</span>
     <span class="n">bash_command</span><span class="o">=</span><span class="s1">&#39;date&#39;</span><span class="p">,</span>
@@ -419,7 +419,7 @@ something like this:</p>
 <span class="n">dag</span> <span class="o">=</span> <span class="n">DAG</span><span class="p">(</span>
     <span class="s1">&#39;tutorial&#39;</span><span class="p">,</span> <span class="n">default_args</span><span class="o">=</span><span class="n">default_args</span><span class="p">,</span> <span class="n">schedule_interval</span><span class="o">=</span><span class="n">timedelta</span><span class="p">(</span><span class="mi">1</span><span class="p">))</span>
 
-<span class="c1"># t1, t2 and t3 are examples of tasks created by instatiating operators</span>
+<span class="c1"># t1, t2 and t3 are examples of tasks created by instantiating operators</span>
 <span class="n">t1</span> <span class="o">=</span> <span class="n">BashOperator</span><span class="p">(</span>
     <span class="n">task_id</span><span class="o">=</span><span class="s1">&#39;print_date&#39;</span><span class="p">,</span>
     <span class="n">bash_command</span><span class="o">=</span><span class="s1">&#39;date&#39;</span><span class="p">,</span>

[2/7] incubator-airflow-site git commit: New version of the docs

[ma...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/searchindex.js
----------------------------------------------------------------------
diff --git a/searchindex.js b/searchindex.js
index 97adc3c..3d2f62d 100644
--- a/searchindex.js
+++ b/searchindex.js
@@ -1 +1 @@
-Search.setIndex({envversion:47,filenames:["cli","code","concepts","configuration","faq","index","installation","license","plugins","profiling","project","scheduler","security","start","tutorial","ui"],objects:{"airflow.contrib":{hooks:[1,1,0,"-"],operators:[1,1,0,"-"]},"airflow.contrib.executors.mesos_executor":{MesosExecutor:[1,0,1,""]},"airflow.contrib.hooks":{BigQueryHook:[1,0,1,""],CloudantHook:[1,0,1,""],FTPHook:[1,0,1,""],GoogleCloudStorageHook:[1,0,1,""],SSHHook:[1,0,1,""],VerticaHook:[1,0,1,""]},"airflow.contrib.hooks.BigQueryHook":{get_conn:[1,2,1,""],get_pandas_df:[1,2,1,""],get_service:[1,2,1,""],insert_rows:[1,2,1,""]},"airflow.contrib.hooks.CloudantHook":{db:[1,2,1,""]},"airflow.contrib.hooks.FTPHook":{close_conn:[1,2,1,""],create_directory:[1,2,1,""],delete_directory:[1,2,1,""],delete_file:[1,2,1,""],describe_directory:[1,2,1,""],get_conn:[1,2,1,""],list_directory:[1,2,1,""],retrieve_file:[1,2,1,""],store_file:[1,2,1,""]},"airflow.contrib.hooks.GoogleCloudStorageHook":
 {download:[1,2,1,""],get_conn:[1,2,1,""],upload:[1,2,1,""]},"airflow.contrib.hooks.SSHHook":{Popen:[1,2,1,""],check_output:[1,2,1,""],tunnel:[1,2,1,""]},"airflow.contrib.hooks.VerticaHook":{get_conn:[1,2,1,""]},"airflow.contrib.hooks.gcs_hook":{GoogleCloudStorageHook:[1,0,1,""]},"airflow.contrib.operators":{QuboleOperator:[1,0,1,""],SSHExecuteOperator:[1,0,1,""],VerticaOperator:[1,0,1,""],VerticaToHiveTransfer:[1,0,1,""]},"airflow.contrib.operators.bigquery_operator":{BigQueryOperator:[1,0,1,""]},"airflow.contrib.operators.bigquery_to_gcs":{BigQueryToCloudStorageOperator:[1,0,1,""]},"airflow.contrib.operators.gcs_download_operator":{GoogleCloudStorageDownloadOperator:[1,0,1,""]},"airflow.contrib.operators.hipchat_operator":{HipChatAPIOperator:[1,0,1,""],HipChatAPISendRoomNotificationOperator:[1,0,1,""]},"airflow.executors":{CeleryExecutor:[1,0,1,""],LocalExecutor:[1,0,1,""],SequentialExecutor:[1,0,1,""]},"airflow.hooks":{DbApiHook:[1,0,1,""],DruidHook:[1,0,1,""],HiveCliHook:[1,0,1,"
 "],HiveMetastoreHook:[1,0,1,""],HiveServer2Hook:[1,0,1,""],HttpHook:[1,0,1,""],MsSqlHook:[1,0,1,""],MySqlHook:[1,0,1,""],PostgresHook:[1,0,1,""],PrestoHook:[1,0,1,""],S3Hook:[1,0,1,""],SqliteHook:[1,0,1,""],WebHDFSHook:[1,0,1,""]},"airflow.hooks.DbApiHook":{bulk_dump:[1,2,1,""],bulk_load:[1,2,1,""],get_conn:[1,2,1,""],get_cursor:[1,2,1,""],get_first:[1,2,1,""],get_pandas_df:[1,2,1,""],get_records:[1,2,1,""],insert_rows:[1,2,1,""],run:[1,2,1,""]},"airflow.hooks.DruidHook":{construct_ingest_query:[1,2,1,""],get_conn:[1,2,1,""],load_from_hdfs:[1,2,1,""]},"airflow.hooks.HiveCliHook":{load_file:[1,2,1,""],run_cli:[1,2,1,""],test_hql:[1,2,1,""]},"airflow.hooks.HiveMetastoreHook":{check_for_named_partition:[1,2,1,""],check_for_partition:[1,2,1,""],get_databases:[1,2,1,""],get_metastore_client:[1,2,1,""],get_partitions:[1,2,1,""],get_table:[1,2,1,""],get_tables:[1,2,1,""],max_partition:[1,2,1,""],table_exists:[1,2,1,""]},"airflow.hooks.HiveServer2Hook":{get_pandas_df:[1,2,1,""],get_records:
 [1,2,1,""]},"airflow.hooks.HttpHook":{get_conn:[1,2,1,""],run:[1,2,1,""],run_and_check:[1,2,1,""]},"airflow.hooks.MsSqlHook":{get_conn:[1,2,1,""]},"airflow.hooks.MySqlHook":{bulk_load:[1,2,1,""],get_conn:[1,2,1,""]},"airflow.hooks.PrestoHook":{get_conn:[1,2,1,""],get_first:[1,2,1,""],get_pandas_df:[1,2,1,""],get_records:[1,2,1,""],run:[1,2,1,""]},"airflow.hooks.S3Hook":{check_for_bucket:[1,2,1,""],check_for_key:[1,2,1,""],check_for_prefix:[1,2,1,""],check_for_wildcard_key:[1,2,1,""],get_bucket:[1,2,1,""],get_conn:[1,2,1,""],get_key:[1,2,1,""],get_wildcard_key:[1,2,1,""],list_keys:[1,2,1,""],list_prefixes:[1,2,1,""],load_file:[1,2,1,""],load_string:[1,2,1,""]},"airflow.hooks.SqliteHook":{get_conn:[1,2,1,""]},"airflow.hooks.WebHDFSHook":{check_for_path:[1,2,1,""],get_conn:[1,2,1,""],load_file:[1,2,1,""]},"airflow.macros":{ds_add:[1,3,1,""],ds_format:[1,3,1,""],hive:[1,1,0,"-"],random:[1,3,1,""]},"airflow.macros.hive":{closest_ds_partition:[1,3,1,""],max_partition:[1,3,1,""]},"airflow.
 models":{BaseOperator:[1,0,1,""],Connection:[1,0,1,""],DAG:[1,0,1,""],DagBag:[1,0,1,""],TaskInstance:[1,0,1,""]},"airflow.models.BaseOperator":{clear:[1,2,1,""],dag:[1,4,1,""],detect_downstream_cycle:[1,2,1,""],downstream_list:[1,4,1,""],execute:[1,2,1,""],get_direct_relatives:[1,2,1,""],get_flat_relatives:[1,2,1,""],get_task_instances:[1,2,1,""],has_dag:[1,2,1,""],on_kill:[1,2,1,""],post_execute:[1,2,1,""],pre_execute:[1,2,1,""],prepare_template:[1,2,1,""],render_template:[1,2,1,""],render_template_from_field:[1,2,1,""],run:[1,2,1,""],schedule_interval:[1,4,1,""],set_downstream:[1,2,1,""],set_upstream:[1,2,1,""],upstream_list:[1,4,1,""],xcom_pull:[1,2,1,""],xcom_push:[1,2,1,""]},"airflow.models.Connection":{extra_dejson:[1,4,1,""]},"airflow.models.DAG":{add_task:[1,2,1,""],add_tasks:[1,2,1,""],cli:[1,2,1,""],concurrency_reached:[1,4,1,""],crawl_for_tasks:[1,2,1,""],create_dagrun:[1,2,1,""],filepath:[1,4,1,""],folder:[1,4,1,""],get_template_env:[1,2,1,""],is_paused:[1,4,1,""],latest
 _execution_date:[1,4,1,""],normalize_schedule:[1,2,1,""],run:[1,2,1,""],set_dependency:[1,2,1,""],sub_dag:[1,2,1,""],subdags:[1,4,1,""],tree_view:[1,2,1,""]},"airflow.models.DagBag":{bag_dag:[1,2,1,""],collect_dags:[1,2,1,""],dagbag_report:[1,2,1,""],get_dag:[1,2,1,""],kill_zombies:[1,2,1,""],process_file:[1,2,1,""],size:[1,2,1,""]},"airflow.models.TaskInstance":{are_dependencies_met:[1,2,1,""],are_dependents_done:[1,2,1,""],clear_xcom_data:[1,2,1,""],command:[1,2,1,""],current_state:[1,2,1,""],error:[1,2,1,""],evaluate_trigger_rule:[1,2,1,""],is_premature:[1,2,1,""],is_queueable:[1,2,1,""],is_runnable:[1,2,1,""],key:[1,4,1,""],pool_full:[1,2,1,""],ready_for_retry:[1,2,1,""],refresh_from_db:[1,2,1,""],run:[1,2,1,""],xcom_pull:[1,2,1,""],xcom_push:[1,2,1,""]},"airflow.operators":{BashOperator:[1,0,1,""],BranchPythonOperator:[1,0,1,""],DummyOperator:[1,0,1,""],EmailOperator:[1,0,1,""],ExternalTaskSensor:[1,0,1,""],GenericTransfer:[1,0,1,""],HdfsSensor:[1,0,1,""],Hive2SambaOperator:[1,
 0,1,""],HiveOperator:[1,0,1,""],HivePartitionSensor:[1,0,1,""],HiveToDruidTransfer:[1,0,1,""],HiveToMySqlTransfer:[1,0,1,""],HttpSensor:[1,0,1,""],MetastorePartitionSensor:[1,0,1,""],MsSqlOperator:[1,0,1,""],MsSqlToHiveTransfer:[1,0,1,""],MySqlOperator:[1,0,1,""],MySqlToHiveTransfer:[1,0,1,""],PostgresOperator:[1,0,1,""],PrestoCheckOperator:[1,0,1,""],PrestoIntervalCheckOperator:[1,0,1,""],PrestoValueCheckOperator:[1,0,1,""],PythonOperator:[1,0,1,""],S3KeySensor:[1,0,1,""],S3ToHiveTransfer:[1,0,1,""],ShortCircuitOperator:[1,0,1,""],SimpleHttpOperator:[1,0,1,""],SlackAPIOperator:[1,0,1,""],SlackAPIPostOperator:[1,0,1,""],SqlSensor:[1,0,1,""],TimeSensor:[1,0,1,""],TriggerDagRunOperator:[1,0,1,""],WebHdfsSensor:[1,0,1,""]},"airflow.operators.BashOperator":{execute:[1,2,1,""]},"airflow.operators.SlackAPIOperator":{construct_api_call_params:[1,2,1,""],execute:[1,2,1,""]},"airflow.operators.docker_operator":{DockerOperator:[1,0,1,""]},"airflow.operators.sensors":{BaseSensorOperator:[1,0,1
 ,""]},airflow:{executors:[1,1,0,"-"],hooks:[1,1,0,"-"],macros:[1,1,0,"-"],models:[1,1,0,"-"],operators:[1,1,0,"-"]}},objnames:{"0":["py","class","Python class"],"1":["py","module","Python module"],"2":["py","method","Python method"],"3":["py","function","Python function"],"4":["py","attribute","Python attribute"]},objtypes:{"0":"py:class","1":"py:module","2":"py:method","3":"py:function","4":"py:attribute"},terms:{"00am":1,"01t23":11,"10gb":1,"10pm":2,"128m":1,"2am":1,"5gb":1,"__class__":2,"__doc__":2,"__init__":[1,2],"__name__":[2,8],"_cmd":3,"_host":12,"abstract":[1,2,5,8],"boolean":[1,2],"byte":[1,4],"case":[1,2,4,11,12,14],"char":1,"class":[0,1,2,6,7,8,14],"default":[],"export":[1,13],"final":1,"float":1,"function":[],"import":[],"int":1,"long":1,"new":[1,2,4,8,11,12],"return":[0,1,2,8],"short":1,"static":[1,5,8],"super":[1,12],"switch":[3,12],"true":[1,2,4,12,14],"try":[1,2,3],"var":1,"while":[1,2,4,5,7,11,13,14,15],abbrevi:2,abil:2,abl:[1,3,4,12,13,14],about:[1,2,3,7,10,11,14]
 ,abov:[2,3,7,12,15],absenc:2,absolut:2,accept:[1,5,7],access:[0,1,2,3,8,12],access_logfil:0,access_token:15,accord:12,accordingli:2,account:[1,12],acquir:1,across:[3,15],act:[1,2,7,14],action:[1,15],activ:[1,2,4,8,11,12,14],actual:[1,2,4,14],acycl:[1,2,5],add:[1,2,3,7,12],add_task:1,addendum:7,addit:[],addition:12,addprinc:12,adhoc:[],adjust:[3,12],admin:[2,3,4,8,9,12,13],admin_view:8,advanc:[1,14],advantag:12,advis:[1,7],affect:2,after:[1,2,4,11],afterward:1,against:[1,2,4,7,12,14],agari:10,aggregate_db_message_job:2,aggregate_db_message_job_cmd:2,ago:4,agre:7,agreement:[2,7],airbnb:[1,2,10,14],airflow:[],airflow__:3,airflow__core__sql_alchemy_conn:3,airflow_config:3,airflow_conn_:[2,3],airflow_conn_postgres_mast:2,airflow_hom:[3,4,8,13],airflow_set:2,airflow_tmp_dir:1,airflowignor:1,airflowplugin:8,airflowtestplugin:8,aiyzngq:1,alchemi:3,alert:[1,2,8],align:4,all:[0,1,2,3,4,5,6,7,11,12,14],all_db:6,all_don:[1,2],all_fail:[1,2],all_success:[1,2],alleg:7,alloc:1,allow:[0,1,2,3,4,5,8
 ,9,11,12,13,14,15],allow_large_result:1,allowed_st:1,allowed_team:12,allowinsert:1,alon:7,along:[1,4,7,14],alongsid:[1,7,11],alreadi:1,alright:14,also:[1,2,3,4,6,7,8,11,12,14],alter:[1,2,3,4,12],altern:[1,11],altert:1,although:12,alwai:[1,2,15],amazon:3,amount:[1,2],analys:15,analyz:2,anand:10,anchor:1,ani:[1,2,3,4,7,14,15],annot:7,announc:10,anomali:8,anoth:[1,2,4,10],ansibl:3,anyon:12,anyth:[2,14],anytim:2,anywher:[1,2],aoen:10,apach:[],api:[],api_call_param:1,api_kei:15,api_param:1,api_vers:1,apikei:15,apiv2:1,app:1,app_id:1,appar:1,appear:[1,2,7],append:[1,3],appendix:7,appli:[1,2,6,7,11],applic:[1,7,8,9,12],appropri:[2,7,12],arbitrari:[2,5],architectur:5,archiv:[1,7],archive1:1,archive2:1,are_dependencies_met:1,are_dependents_don:1,aren:[1,2],arg:1,argument:[],aris:7,around:[1,5,14],arrai:[1,5,8],arthur:10,artwr:10,ascii:1,asf:5,assert:7,asset:1,assign:[],associ:[1,2,7,11],assum:[1,2,3,7,14],asterisk:12,async:6,atom:2,attach:[1,2,7],attach_to:1,attack:12,attempt:[0,1],attent:[1
 ,4],attr:1,attribut:[0,1,2,7],audit:8,auth:[1,12],auth_backend:12,auth_token:1,authen:12,authent:[],authmechan:1,author:[1,3,5,7,12,14,15],authorship:7,auto:4,autocommit:1,automat:[1,2,3],avail:[1,2,3,7,8,9],averag:1,avoid:2,awai:1,awar:2,azkaban:5,back:[1,4,8],backend:[],backfil:[],background:[1,14],backup:3,bag:1,bag_dag:1,balanc:2,bar:[1,2],bare:14,base64:4,base:[1,2,3,4,7,8,11],base_executor:[1,8],base_hook:[1,8],base_url:1,basedn:12,baseexecutor:[1,8],basehook:[1,2,8],baseview:8,bash:[1,2,14],bash_1:2,bash_command:[1,2,14],bash_command_to_run:3,bash_oper:[1,14],bash_queu:14,bashoper:[1,2,14],basi:7,basic:[2,6,8,9,14],bcrypt:12,beauchemin:10,becaus:[1,2],becom:[1,2,5,8,14],beelin:1,been:[1,2,3,7,11,12],befor:[0,1,2,4,12],begin:[2,11],behalf:7,behav:1,behavior:[1,2],behind:[1,3,6,11],belong:[1,12],below:[1,7,8,13,14],benefici:7,best:[1,10],better:14,between:[1,2,14],beyond:[],bigqueri:1,bigquery_conn_id:1,bigquery_default:1,bigquery_oper:1,bigquery_to_gc:1,bigqueryhook:1,bigquery
 oper:1,bigquerytocloudstorageoper:1,bin:2,bind:7,bind_password:12,bind_us:12,bit:3,blob:[1,14],block:[1,2,14,15],blog:[7,10],blueprint:8,bodi:1,boilerpl:7,bolk:10,bolkedebruin:10,bonu:1,bool:1,boot:3,both:[1,4],bother:14,boto:1,bottleneck:15,boundary_queri:1,box:[1,3,13],bql:1,bracket:[7,14],branch:[],branch_fals:2,branchpythonoper:[1,2],broker:[0,3],broker_api:0,broker_url:3,brought:10,brows:[2,4],bruin:10,bucket:[1,3],bucket_kei:1,bucket_nam:1,buffer:1,bug:3,build:[],built:[1,2,3,5,8,9,14],bulk:[4,15],bulk_dump:1,bulk_load:1,bump:2,bundl:1,burn:0,but_this_dag_will_not:2,c12318391:1,cacert:12,cach:12,call:[1,2,3,14],callabl:1,callback:12,can:[1,2,3,4,5,6,8,9,11,12,13,14,15],cannot:[1,2,7,14],capabl:8,capac:2,card:1,care:[0,3],carri:[2,7],cast:1,cat:1,categori:8,caus:[4,7],celeri:[],celery_result_backend:3,celeryexecutor:[1,2,3,4,6,11],cell:1,central:2,cert:1,certain:[1,2],certif:1,cfg:[1,2,3,4,11,12,13,14],chain:2,chang:[3,5,7,11,12,13,14],channel:[1,10],charact:7,character:2,charg
 :7,charset:1,chart:[],chat:10,check:[0,1,2,4],check_for_bucket:1,check_for_kei:1,check_for_named_partit:1,check_for_partit:1,check_for_path:1,check_for_prefix:1,check_for_wildcard_kei:1,check_head:1,check_oper:1,check_output:1,checkoper:1,chef:[3,10],child:[1,2],child_dag:2,child_dag_nam:2,chmod:12,choic:[0,1,14],choos:[1,2,7],chri:10,circuit:1,claim:7,clariti:5,claus:1,clean:1,cleanup:1,clear:[0,1,2,4,11,15],clear_xcom_data:1,cli:[1,2,3,4,11],click:[4,12,14,15],client:[1,6,12],client_id:12,client_secret:12,close:[1,4,12],close_conn:1,closest:1,closest_ds_partit:1,cloud:[1,3,6],cloudant:[1,6],cloudant_conn_id:1,cloudant_default:1,cloudanthook:1,cluster:[],cluster_label:1,cmd:1,cmdline:1,code:[],col:1,collabor:5,collect:[1,2,8],collect_dag:1,color:1,column:1,com:[1,2,12,14],combin:[2,7],come:[1,11],comma:[0,1,2],command_typ:1,comment:7,commerci:7,commit:[1,10,12],commit_everi:1,committ:[],common:[2,3,4,7,10,14],commonli:1,compani:8,compar:5,compil:[4,7],complet:[1,2,3,5],complex:[1,2
 ,5,8],compli:7,complianc:7,complic:14,compon:8,compos:[2,12,14],comprehens:15,compress:1,comput:[1,7],conceptu:2,concern:2,concur:4,concurr:[0,1,4],concurrency_reach:1,condit:[1,2,6,7],conf:[0,1,3],config:[8,12],configur:[],confirm:[0,4,11],confus:[1,2,4],conjunct:[0,2,13],conn_id:[1,2,3],conn_typ:1,connect:[],connect_timeout:1,connecton:9,consequenti:7,consid:[1,2,3],consider:1,consist:[1,2,5,7],conspicu:7,constant:[1,2],constitu:2,constitut:7,constraint:1,constru:7,construct:1,construct_api_call_param:1,construct_ingest_queri:1,constructor:[1,14],contact:[3,7],contain:[1,2,3,7,11,12,14,15],content:[],context:[],continu:[1,5],contract:7,contrib:[1,2,12],contributor:[7,10],contributori:7,control:[1,2,7,15],conveni:[1,2],convent:2,convers:7,cool:1,coordin:1,copi:[1,3,7,12],copyright:[7,12],core:[],correct:2,correpsond:2,correspond:[1,2,3,4],could:[1,2,14],count:1,counterclaim:7,cours:12,cover:11,cpu:[1,8],craft:1,crawl:[1,11],crawl_for_task:1,creat:[1,2,3,4,8,9,11,12,13,14,15],create
 _dagrun:1,create_directori:1,creation:1,credenti:12,credit:12,criccomini:10,criteria:[1,2],criterion:1,critic:[1,2],cron:[1,4,11],cross:[2,7,14],crt:12,crypto:[3,4,6],csv:1,curli:14,current:[1,15],current_st:1,cursor:1,custom:8,customari:7,cycl:[1,2,14],daemon:[0,1,3],dag:[],dag_1:2,dag_2:2,dag_fold:[1,2],dag_id:[0,1,2,14],dag_md:2,dag_run:1,dag_run_obj:1,dag_stat:0,dagbag:[0,1],dagbag_report:1,dagfil:2,dagrun:[0,1,4],dagrun_timeout:1,dags_fold:3,dai:[1,11,13,14],daili:[1,4,11],damag:7,dan:[1,10],data:[],data_profiler_filt:12,databas:[0,1,2,3,5,6,8,9,13,14],datafram:1,dataset:1,datasourc:1,date:[0,1,2,7,11,14],date_filter_column:1,datestamp:1,datetim:[1,2,4,11,14],dateutil:1,davydov:10,days_back:1,db_tabl:1,db_tap_id:1,db_update_kei:1,db_update_mod:1,dbapi_hook:1,dbapihook:1,dbexportcmd:1,dbimportcmd:1,dbtap_id:1,dbtapquerycmd:1,deal:4,debug:[0,1,14],decid:[1,3],decis:5,declar:1,deep:1,def:[1,2,8],default_arg:[1,2,4,14],default_login:12,default_queu:2,defend:7,defer:2,deferred_op:2,
 defin:[1,2,3,4,5,7,8,11,12,14],definit:[],del:1,delai:[1,4],delegate_to:1,delet:[1,2,11,15],delete_directori:1,delete_fil:1,deliber:[1,7],delimit:[0,1,2],demonstr:2,dep:1,depend:[],depends_on_past:[0,1,2,4,14],deploi:8,deploy:8,depth:14,deriv:[1,3,7,8],descend:2,describ:[1,2,7],describe_directori:1,descript:[1,2,7],deseri:[0,1,2],deserialize_json:2,design:[2,7,11],desir:[1,14],destin:1,destination_cloud_storage_uri:1,destination_conn_id:1,destination_dataset_t:1,destination_filepath:1,destination_t:1,detail:[1,2,4,12,15],detect:[1,8],detect_downstream_cycl:1,determin:[1,2,7],dev:[3,6,10],devel:6,devel_hadoop:6,develop:[0,1,12,14],deviat:1,dialog:11,dict:[0,1,2],dictionari:[1,2,14],did:14,die:2,differ:[1,2,7,8,14,15],dimens:1,direct:[1,2,3,5,7],directli:[1,2,4,9],directori:[0,1,2,3,12,14],dirti:13,disabl:1,disclaim:[],discuss:7,disk:1,displai:[3,7,11,14],disregard:[1,14],distinct:1,distribut:[1,2,6,7],do_pickl:0,doc:[1,2,4,14],doc_json:2,doc_md:2,doc_rst:2,doc_yaml:2,docker:[1,10],do
 cker_oper:1,docker_url:1,dockeroper:[1,2],document:[],doe:[0,1,2,3,4,5,6,7,12,14],does_not_exist:1,doesn:[1,2,14],domain:12,don:[1,2,3,6,7,11],done:[1,2,11,12,14],donot_pickl:[0,1],door:2,dot:[1,2],doubl:[1,3,14],down:[0,2],download:[1,3],downstream:[0,1,2,11],downstream_list:1,downstream_task_id:1,drive:3,driven:[1,8],driver:3,drop:[1,8],druid:[1,6],druid_datasourc:1,druid_ingest_conn_id:1,druid_ingest_default:1,druid_query_conn_id:1,druid_query_default:1,druidhook:1,dry:0,dry_run:0,ds_add:[1,14],ds_format:1,ds_nodash:1,dttm:1,dubiou:1,dummi:[1,2,8],dummy_1:2,dummy_oper:2,dummy_task:2,dummyoper:[1,2],dump:1,durat:[],dure:[3,15],dynam:[1,2,4,5],each:[1,2,4,7,10,11,14],earlier:14,easi:[2,3,5,8,9,12,15],easier:[1,7],easiest:6,easili:[1,2,5,9,14],echo:[2,14],ecosystem:8,edit:[3,15],editori:7,effect:[1,2],effici:1,effort:5,egg:3,either:[1,2,3,7,12,13],elabor:7,elaps:1,elast:1,electron:7,eleg:5,element:3,els:13,elsewher:2,email:[1,2,10,12,14],email_on_failur:[1,14],email_on_retri:[1,14],
 emailoper:[1,2],emit:[2,14],empti:[1,2],enabl:[],enclos:7,encod:[1,4],encrypt:[1,3,4,6,12],encrypt_s3_log:3,end:[1,7,11],end_dat:[0,1,14],endors:5,endpoint:1,enforc:[2,4],engin:[1,4,5],enterpris:[],entir:[0,4],entiti:7,entri:[1,11],env:[1,2],environ:[1,2,3,5,6,9,11,14,15],ep_data_pipeline_db_msg_agg:2,equival:[2,6,14],era:4,error:[0,1],error_logfil:0,especi:[2,4,14],essenti:1,etc:[1,2,3,12],eval:1,evalu:[1,2,4,11,14],evaluate_trigger_rul:1,even:[1,2,7,9],event:[2,7,14],eventlet:0,ever:1,everi:[1,2,11,12,14],everyon:14,everyth:[14,15],exactli:[1,15],exampl:[],example1:13,example_bash_oper:13,example_dag:[2,14],example_team_1:12,example_team_2:12,exce:1,except:[1,2,4,7,14],exchang:[2,5],exclud:[0,7],exclude_subdag:0,exclus:[7,11],execut:[0,1,2,3,4,5,7,11,14],execution_d:[0,1,2,14],execution_date_fn:1,execution_delta:1,execution_timeout:[1,2],exercis:7,exhaust:3,exist:[0,1,2,3,4,14],exit:[0,12],expect:[1,2,4,5],expir:1,explain:1,explan:14,explicit:5,explicit_op:2,explicitli:[1,2,7,12,1
 4],export_dir:1,export_format:1,expos:[1,2,3,8,12],express:[1,4,7,11],ext:1,extend:5,extens:[1,5],extern:[],external_dag_id:1,external_task_id:1,external_trigg:1,externaltasksensor:1,extra:[],extra_dejson:1,extra_opt:1,extract:1,extract_queri:1,fact:2,factori:2,fail:[0,1,2,3,4,11,14,15],failur:[1,3,4,7],fairli:13,fall:10,fals:[0,1,3,11,14],familiar:[2,14],faq:[],fast:13,fault:[1,2],featur:[1,2,3,6,8,14,15],feb:12,fee:7,fernet:4,fernet_kei:4,fetch:[1,3],few:[1,3,9,13,14],field:[1,7],field_delimit:1,field_dict:1,fields_terminated_bi:1,fifti:7,figur:[1,2,11],file1:1,file2:1,file:[],filenam:[1,2],filepath:1,filestatu:1,filesystem:3,fill:[1,2,9,12],filter:[0,1,2,12],filter_by_own:12,find:[1,3,4,14,15],fine:14,finish:1,fire:[1,2,3],first:[0,1,2,3,4,10,11,13,14],fit:[1,5,7],fix:[1,11],flag:[1,11],flag_upstream_fail:1,flask:[0,8,9],flask_admin:8,flask_blueprint:8,flask_login:12,flat:1,flavor:8,flexibl:[1,14],flower:[0,3],folder:[1,3,8,11,13,14],follow:[1,2,3,5,7,12,14],foo:[1,2],foo_dag_id:
 4,foobar:3,forc:[0,1],force_pul:1,foreground:0,foreign:1,form:[],format:[1,2,3,7],forward:[1,4],found:[1,3,12],foundat:[5,7,10,13],four:2,framework:[3,8],free:[2,7],frm:1,from:[0,1,2,3,4,5,7,8,9,10,11,12,13,14,15],ftp:1,ftp_conn_id:1,ftp_default:1,ftphook:1,full:[1,10],full_filepath:1,fulli:[4,5,12],functiontyp:1,fundament:14,further:[5,13,14],futur:[1,11],galaxi:8,gantt:[],garag:2,gate:12,gcp_api:6,gcp_api_base_hook:1,gcs_download_oper:1,gcs_hook:1,gender:1,gener:[0,1,2,5,7,8,12,14,15],generictransf:1,get:[],get_bucket:1,get_conn:1,get_connect:2,get_cursor:1,get_dag:1,get_databas:1,get_direct_rel:1,get_first:1,get_flat_rel:1,get_kei:1,get_metastore_cli:1,get_pandas_df:1,get_partit:1,get_partitions_by_filt:1,get_record:1,get_servic:1,get_tabl:1,get_task_inst:1,get_template_context:1,get_template_env:1,get_wildcard_kei:1,gevent:0,ghe:[],ghe_oauth:12,ghost:1,git:3,github:[],github_enterpris:12,github_enterprise_auth:12,githubusercont:1,gitignor:1,gitter:10,give:[2,7],given:[1,2],glanc
 :15,global:[1,2,4],goal:14,goe:[1,4,14],good:2,goodby:2,goodwil:7,googl:[1,3,6],google_cloud_storage_conn_id:1,google_cloud_storage_default:1,googlecloudbasehook:1,googlecloudstoragedownloadoper:1,googlecloudstoragehook:1,govern:7,grab:1,grade:3,grai:1,grant:7,graph:[],great:[2,3],greater:1,green:1,grossli:7,group:[1,3,12],gunicorn:[0,6],hack:1,had:[1,4],hadoop:[6,10,12],hadoop_dependency_coordin:1,hadoopcmd:1,hand:[12,13],handi:10,handl:[1,2,3],happen:[2,11,14],hard:[1,2],harmless:7,has_dag:1,hash:[1,12],hasn:[1,2],have:[1,2,3,4,5,6,7,8,9,11,12,14],haven:[1,2,14],hdf:[1,2,6],hdfs_conn_id:1,hdfs_default:1,hdfs_path:1,hdfscli:1,hdfssensor:1,head:[13,14],header:1,heartbeat:[1,2],heavier:1,hello:[2,8],help:[0,7,8,12],here:[1,2,3,4,6,8,10,11,13,14,15],herebi:7,herein:7,hex:1,hidden:15,hierarchi:[1,14],high:1,highchart:9,higher:1,highli:3,highlight:14,hipchat:1,hipchat_oper:1,hipchatapioper:1,hipchatapisendroomnotificationoper:1,histori:[],hit:[1,2],hive2mysqloper:1,hive2sambaoper:[1,6],
 hive:[1,2,3,6,8,12],hive_cli_conn_id:1,hive_cli_default:1,hive_cli_param:1,hive_conf:1,hive_conn_id:1,hive_t:1,hiveclihook:1,hivecmd:1,hiveconf:1,hiveconf_jinja_transl:1,hivemetastorehook:1,hiveoper:[1,2,3,12],hivepartitionsensor:[1,2],hivesensoroper:1,hiveserver2:1,hiveserver2_conn_id:1,hiveserver2_default:1,hiveserver2hook:1,hivetodruidtransf:1,hivetomysqltransf:1,hold:7,home:13,homogen:3,hope:11,horribl:14,host:[1,3,12],hostnam:[0,1,2,3],hour:[1,2,4,11],hourli:[1,4,11],hous:2,how:[1,2,3,4,7,8,11,14,15],howev:[2,7,12],hql:1,html:[1,8,14],html_content:1,http:[1,2,7,8,12,14],http_conn_id:1,http_default:1,httphook:1,httpoper:2,httpsensor:1,human:1,icon:1,icon_url:1,idea:[],idempot:1,identif:7,identifi:[0,1,7,14,15],ignor:[0,1],ignore_depend:[0,1],ignore_depends_on_past:[0,1],ignore_first_depends_on_past:[0,1],iii:7,imag:1,immedi:[1,2],imper:3,imperson:12,implement:[1,2],impli:[1,7],improv:7,impyla:1,inabl:7,inact:4,inc:7,incident:7,includ:[0,1,2,3,7,11,14],include_adhoc:[0,1],include
 _downstream:1,include_exampl:1,include_prior_d:1,include_queu:1,include_upstream:1,inclus:7,incorpor:7,incur:7,indemn:7,indemnifi:7,independ:[1,2],index:1,indic:[1,2,5,7],indirect:7,individu:[1,7,11,14],ineffici:1,infer:[1,2],inferred_op:2,infil:1,infin:5,info:1,inform:[1,2,3,7,11,12,14],infrastructur:[1,5],infring:7,ingest:1,inherit:[1,14],init:3,initctl:3,initdb:[0,3,13],initi:[0,3,12,13],initialis:1,inject:8,inlin:1,input:1,input_format:1,insecur:12,insecurecli:1,insert:[1,2],insert_row:1,insid:[1,2],inspect:[11,13],instal:[],instanc:[],instanti:[],instati:14,instead:[0,1,4],insteadnhttp:1,institut:7,instruct:2,insur:1,integr:[],integrate_plugin:[],intention:7,inter:2,interact:[1,2,3,8,9],interest:[1,14],intergr:8,intern:0,interpret:[1,2],interv:[1,4],intervalcheckoper:1,intoth:1,introduc:4,intuit:14,invari:[1,2],invok:1,ipython:2,irrevoc:7,is_paus:1,is_prematur:1,is_queueabl:1,is_runn:1,ish:1,isn:[1,2,4],isoformat:1,issu:[1,2,5,7,10],item:2,iter:1,itself:[1,2,12,14],januari:[7,1
 1],jar:1,java:1,jdbc:[1,6],jeremiah:10,jinja2:1,jinja:[],jinja_env:1,jlowin:10,job:[0,1,2,3,4,11,13,15],job_id:1,jobtrack:1,join:[2,10],json:[0,1,2],jupyt:2,just:[0,1,2,4,8,14],kadmin:12,kapil:3,keep:[],kei:[0,1,2,3,4,12,15],kerber:[6,12],kerbero:[],kernel:1,key1:1,key_fil:1,keytab:[0,12],keyword:1,kick:[3,11],kill:[1,2],kill_zombi:1,kind:7,know:[3,14],knowledg:8,known:6,known_host:1,kruyk:10,kwarg:1,kwd:1,l447:1,lab:1,label:[1,9],lai:13,lambda:[1,2],land:[1,8],languag:[1,7,14],larg:1,larger:1,last:1,late:15,later:9,latest:[0,1,6,14],latest_d:1,latest_execution_d:1,latin_stat:1,launch:12,law:7,lawsuit:7,layout:14,ldap3:12,ldap:[],ldap_auth:12,ldap_ca:12,lead:[1,2],lean:5,least:[1,2,7],leav:1,led:2,left:[2,12],legaci:4,legal:7,len:1,leo:10,less:1,lesson:10,let:[0,1,2,3,9,14,15],level:[1,2,5,11,12],leverag:[1,2,14],liabil:7,liabl:7,lib:[1,2,3],libffi:3,librari:[1,2,3,5,8,14],libz:2,licens:[],licensor:7,life:2,light:[1,2],lighter:1,lightweight:2,like:[1,2,3,6,8,12,14],limit:[],link:[],
 list:[0,1,2,3,4,6,7,8,10,12,14,15],list_dag:[0,4,14],list_directori:1,list_kei:1,list_prefix:1,list_task:[0,4,14],listen:2,lit:1,liter:1,litig:7,live:1,load:[0,1,2],load_fil:1,load_from_hdf:1,load_str:1,local:[0,1,2,3,12,14],local_executor:1,local_full_path_or_buff:1,local_infil:1,local_port:1,localexecutor:[0,1,2,3,11],localhost:2,locat:[0,1,2,3,12,14],lock:1,lock_for_upd:1,log:[],log_fil:0,logfil:0,loggin:1,loggingmixin:1,logic:[1,2,14],login:[1,2,3,12],logist:8,longer:4,look:[0,1,2,4,5,8,10,11,14],loss:7,lot:11,lowin:10,machin:[1,2,3,12],made:[1,2,7],mai:[1,2,3,4,7,11,14],mail:[7,10],main:[1,2,4,8],main_dag:2,maintain:[1,5,10],make:[1,2,3,4,5,7,12,14,15],malfunct:7,malik:3,man:6,manag:[],mani:[0,1,2,4,8,14,15],manner:5,manual:[1,2,3,11],map:1,mapr:1,mark:[0,1,2,7,11],mark_success:[0,1,11],markdown:2,markup:1,master:[1,2,14],match:[0,1,2],matter:1,max:[1,10],max_active_run:[1,4],max_partit:1,maxim:10,maxime_beauchemin:0,maximum:1,mayb:2,mean:[1,2,3,4,7,12,14],meaning:1,meant:[1,4]
 ,mechan:[1,7,12],media:7,medium:7,meet:[1,7],mem_limit:1,member:12,memberof:12,memori:1,menu:[],menu_link:8,menulink:8,merchant:7,mere:7,merg:14,meso:[],mesos_executor:1,mesosexecutor:[1,3,11],messag:[1,2,5],message_format:1,met:[1,3,4,11],metadata:[],metastor:1,metastore_conn_id:1,metastore_default:1,metastore_mysql:1,metastorepartitionsensor:1,method:[1,2],metric:[1,8],metric_spec:1,metrics_threshold:1,microsoft:[1,6],midnight:[4,11],might:[1,2,4,11,14],migrat:12,mime:1,mime_typ:1,min:4,mind:[],minimum:[1,6,11],minut:[2,4,11,14],mismatch:2,miss:[1,2],mistercrunch:10,mix:14,mkdir:2,mlsd:1,mode:[0,1,14],modif:7,modifi:[1,2,3,7,9],modul:[],modular:5,moment:4,monitor:[2,3,5,8,11,15],monospac:2,month:[4,11],monthli:[4,11],more:[1,2,3,5,7,8,11,12,14,15],morn:11,most:[1,2,12,14,15],mostli:[1,5,11],mount:[1,3],move:[1,2,4,5],msg:1,mssql:[1,6],mssql_conn_id:1,mssql_default:1,mssqlhook:1,mssqloper:1,mssqltohivetransf:1,much:[1,2,11],multi:[],multipart:1,multipart_byt:1,multipl:[1,2],multipl
 i:1,multiprocess:1,must:[1,2,3,4,7,12,14],mutat:2,my_conn_str:3,my_dag1:2,my_dag2:2,my_dag:2,my_databas:1,my_dict_var:1,my_funct:2,my_param:14,my_tabl:1,my_useful_packag:2,my_var:1,myarg:1,mypackag:12,mys3conn:3,mysql:[1,2,3,6],mysql_conn_id:1,mysql_default:1,mysql_postoper:1,mysql_preoper:1,mysql_tabl:1,mysqldb:1,mysqlhook:1,mysqloper:[1,3],mysqltohivetransf:1,name:[1,2,3,7,8,12],namedhivepartitionsensor:1,namespac:[1,8],nav:12,navig:12,ndfor:14,necessarili:[5,7],need:[1,2,3,4,5,6,8,11,12,13,14],neg:[1,11],neglig:7,neighbour:1,nest:[1,14],network:[1,12],network_mod:1,never:[0,1,4],new_user_email:12,new_user_nam:12,newli:5,newobject:1,next:[],nhere:1,nice:2,night:2,nlst:1,no_confirm:0,no_host_key_check:1,node:[0,1,2],non:[1,3,7],none:[1,2,8,11],noram:12,norandkei:12,normal:[1,2,7],normalize_schedul:1,notat:1,note:[],notebook:2,noth:[1,7],notic:[2,7,14],notif:1,notifi:1,notwithstand:7,nov:10,now:[1,2,3,4,12,14],nuanc:2,num:1,num_run:0,num_shard:1,number:[0,1,2,3,5],oauth2:12,oauth_ca
 llback_rout:12,oauth_key_from_github_enterpris:12,oauth_secret_from_github_enterpris:12,obj:1,object:[0,1,2,3,4,7,8,11,14],objectclass:12,oblig:7,observ:1,obtain:7,occur:[1,2],occurr:1,octet:1,off:[1,3,11],offer:[7,8],offici:[1,10],offset:1,often:[1,2],on_failure_callback:1,on_kil:1,on_retry_callback:1,on_success_callback:1,onboard:4,onc:[1,2,3,4,11,14],one_fail:[1,2],one_success:[1,2],onli:[0,1,2,3,4,6,7,11,12,13],only_fail:0,only_if_upd:1,only_run:0,oozi:5,op1:2,op2:2,op3:2,op4:2,op_arg:1,op_kwarg:1,open:[1,2,4,10,12],oper:[],oppos:[1,2],optim:1,option:[],orchestr:[1,5],order:[1,2,3,11,12],org:[1,7,8,10,14],organ:[2,8],organiz:12,origin:7,orm:1,other:[1,2,3,5,7,9,11,14],otherwis:[1,7,14],our:14,out:[],outag:8,outgrow:13,outlier:15,output:[1,14],output_encod:1,output_format:1,outsid:11,outstand:7,over:[0,1,2,3,13,15],overlap:15,overrid:[0,1,14],overridden:[1,4],overview:15,overwhelm:2,overwrit:1,own:[],owner:[1,2,7,12,14],ownership:7,packag:[],package1:2,page:[2,4,7,10,15],pai:[1,4
 ],pair:[1,15],panda:1,parallel:[1,2,3,13],param:[0,1,14],paramet:[0,1,2,3,4,9,14],parameter:[2,5,9],parent:[1,2],parent_dag:[1,2],parent_dag_nam:2,parent_modul:1,pars:[1,4,8,14],part:[1,2,7,9,12],parti:7,particular:7,partit:[1,2],partition_nam:1,partition_spec:1,partli:4,pass:[1,2,4,8,14],pass_valu:1,passwd:15,password:[],password_auth:12,passwordus:12,past:[1,2,4,11,15],patent:7,path2:1,path:[1,2,3],patrick:10,patrickleotardif:10,pattern:[1,2],paus:[0,1],payload:1,pem:1,peopl:[1,8,14],pep:1,percent:7,perfect:2,perform:[0,1,2,3,5,7,15],perhap:[2,15],period:[1,2,4,11,14],permiss:[7,12],perpetu:7,persist:[1,11],perspect:2,pertain:7,phase:8,physic:1,pick:[2,3,9],pickl:[0,1,2,3],picklabl:1,pickle_id:1,pickleabl:1,pid:[0,13],pig:[1,2],pigcmd:1,piggi:8,pigoper:1,pin_100:1,pip:[2,6,13],pipelin:[],pitfal:10,place:[1,2,7,14],placehold:1,plain:[1,3],plan:4,platform:[1,2,5,6],pleas:[2,3,10,12,14],plug:8,plugin:[],pluginexecutor:8,pluginhook:8,pluginoper:8,plugins_manag:8,png:1,point:[1,2,3,4,1
 3,14],poke:1,poke_interv:1,polymorph:1,pool:[],pool_ful:1,popen:1,popul:14,port:[0,1,3,12,13],posit:[0,1],possibl:[0,1,2,3,7,12,13,14,15],post:[1,10],post_execut:1,postgr:[1,2,3,6],postgres_conn_id:1,postgres_default:1,postgres_mast:2,postgreshook:1,postgresoper:1,potenti:2,power:[2,5,7,14],practic:[1,10],pre:4,pre_execut:1,preced:[1,3,14],prefer:[7,11,13],prefix:[1,2,3],preoper:1,prepar:[2,7],prepare_templ:1,prepped_request:1,present:1,preset:11,presto:1,presto_conn_id:1,presto_default:1,prestocheckoper:1,prestocmd:1,prestohook:1,prestointervalcheckoper:1,prestotomysqloper:2,prestovaluecheckoper:1,pretti:14,prevent:[1,2,12],previou:[0,1,2,4,14],previous:[2,4],primari:1,primit:1,princip:[0,2,12],print:[0,1,2,7,14],print_dat:14,print_head:1,prior:1,priorit:2,prioriti:[1,2],priority_weight:[1,2,14],privaci:7,probabl:[3,4,12],problem:2,problemat:2,proce:[1,4],process:[0,1,2,3,5,11,14],process_fil:1,produc:1,product:[1,2,3,5,7,9,11,14],profil:[],program:[1,10],programmat:5,progress:[1,5
 ,14],project:[],promin:7,prompt:0,propag:[1,2],propaget:1,proper:14,properli:[3,4],properti:[1,2,4,12],provid:[1,2,3,7,9,14,15],provide_context:[1,2],proxy_us:[1,12],proxyus:12,psycopg2:3,ptarget:1,publicli:7,publish:1,pull:[1,2],pull_funct:2,puppet:[3,10],pure:2,purpl:1,purpos:[7,14],push:[1,2,8],push_funct:2,pushing_task:2,put:[1,2,8],pwd:2,pydata:1,pyhiv:1,pypi:[6,13],python:[1,2,3,5,6,8,12,14],python_1:2,python_cal:[1,2],python_oper:1,pythonhost:8,pythonoper:[1,2],pythonpath:[3,12],qualif:12,qualifi:12,qualiti:[1,2],qubol:[1,6],qubole_conn_id:1,qubole_default:1,quboleoper:1,queri:[],queu:[1,2,4],queue:[],quick:[],quickli:[9,13,14,15],quit:4,r39132:10,rabbitmq:[3,6],rais:[1,14],ran:0,randkei:12,random:1,randomli:2,rang:[0,1,11,14],rather:2,ratio:1,raw:1,reach:[1,2,3,4],reactiv:4,read:[1,3,4,12,14],readabl:[1,7,12],readi:[1,5,13],readm:3,readthedoc:14,ready_for_retri:1,real:3,realli:[1,12,14],realm:12,reason:[4,7,14],reboot:2,rebuild:0,recap:[],receiv:[1,2,7,11],recent:[1,2],recip
 :10,recipi:7,recogn:1,recommend:[1,3,4,7],record:[0,1,2,14],recreat:1,recurs:1,red:1,redhat:3,redi:3,redirect:0,redistribut:7,redund:14,referenc:[2,14],reflect:[2,5,8,14],refrain:2,refresh:1,refresh_from_db:1,regard:[1,7],regardless:[0,1],regarless:1,regex:[0,1],regex_kei:1,regexp:0,regist:[8,9,12],regular:[1,11],reinit_frequ:12,rel:[1,11,14],relat:[1,2,3,4,6],relationship:2,relativedelta:1,relev:4,reli:[1,12],reliabl:1,remain:[7,12],rememb:[1,14],remot:[1,3,11],remote_base_log_fold:3,remote_full_path:1,remote_host:1,remote_log_conn_id:3,remote_port:1,remov:1,render:[0,1,2,8,14],render_templ:1,render_template_from_field:1,renew:[0,12],repeat:2,repl:12,replac:[1,7,12],report:[0,1],repositori:[3,14],repres:[1,2,4,7,11],represent:[1,15],reproduc:7,reproduct:7,request:[0,1,2],requir:[1,2,3,4,5,6,7,12],resembl:2,reset:0,resetdb:0,resid:2,resolv:1,resourc:[],respect:[0,1,2,14],respons:[1,7],response_check:1,rest:1,restart:[2,3,4],restrict:12,restructuredtext:2,result:[1,2,7,9,14],resum:0,
 retain:7,retri:[1,2,14],retriev:[1,2],retrieve_fil:1,retry_delai:[1,14],return_valu:1,reus:8,review:5,revis:7,riccomini:10,rich:[0,2,5,15],rid:1,right:[1,2,7,9],rightmost:2,risk:7,roadmap:[],role:3,roll:[],room:1,room_id:1,root:[1,2],root_dag:1,round:[1,4],rout:12,routin:[1,2],row:[1,2],royalti:7,rule:[],run:[],run_a:1,run_and_check:1,run_as_own:[1,12],run_cli:1,run_id:[0,1,11],runme_0:13,runnabl:[1,2],rwc:12,s3_conn_id:1,s3_default:1,s3_kei:1,s3connect:1,s3distcp:1,s3filetransferoper:2,s3hook:[1,3],s3keysensor:[1,6],s3prefixsensor:6,s3tohivetransf:1,safe_mod:1,sai:[1,2,4,11],said:2,samaccountnam:12,samba:[1,6],samba_conn_id:1,samba_default:1,same:[1,2,3,7,9,12],sampl:[1,3],sample_s:1,sandbox:3,save:[3,4,9,14],scala:1,scalabl:5,scan:[1,2],scene:[6,11],schedul:[],schedule_interv:[1,2,4,11,14],scheduler_interv:4,scheduler_run:3,schema:[1,2,3],scratch:14,screen:14,screenshot:[],script:[],script_begin_tag:1,script_loc:1,search:[1,2],search_path:3,search_scop:12,second:[1,2,4,14],secret:
 [12,15],section:[1,2,3,4,7,12,13,14],secur:[],see:[1,2,3,4,7,11,12,13,14,15],seen:[3,15],segment:1,select:[1,2,9,11,12],self:[1,8],sell:7,semicolon:1,send:[0,1,2,10],send_room_notif:1,sender:1,sensor:[1,2,4],sensor_queu:2,sent:[1,2,3,7],separ:[1,2,7,14],sequenti:[1,11,13],sequentialexecutor:[1,2,13],serial:[0,1],serv:[0,3,14],serve_log:[0,3],server:[0,1,3,8,12,13,14],servic:[0,1,2,3,6,7,11,12],session:[1,12],set:[],set_depend:[1,14],set_downstream:[1,2,14],set_the_password:12,set_upstream:[1,2,14],setgid:3,setuid:3,setup:[3,12],sever:2,shall:7,shape:11,share:[1,2,3,7,8,9],shell:12,shellcmd:1,ship:[0,1],ship_dag:0,shop:7,shortcircuitoper:1,shortcut:[3,15],shortest:1,should:[1,2,3,4,7,8,11,13,14],shouldn:[1,2],show:[0,1,2,4,8,15],shown:1,sid:10,siddharth:10,side:[1,3],signal:1,similar:[2,3,5],simpl:[1,2,8,9,14],simplehttpoper:1,simpler:14,simplest:12,simpli:[1,2,8,11,12,14],simplifi:1,simul:14,simultan:2,sinc:[1,3,13,14],sing:1,singl:[0,1,2,14],site:12,size:[1,2],skew:8,skip:[0,1,2],s
 la:[],sla_miss_callback:1,slack:[1,6],slack_oper:1,slackapioper:1,slackapipostoper:[1,6],slackoper:2,slave:3,sleep:14,slightli:5,slot:[1,2],slowli:5,slug:12,small:[1,2],smaller:1,smallish:1,snap:5,sock:1,soft_fail:1,softwar:[5,7,10],sole:7,solut:5,some:[1,2,3,4,11,14,15],somehow:3,someth:[2,12,14],sometim:[1,2,14],somewhat:14,somewher:13,soon:[1,2,3,4,11],sort:2,sound:[2,14],sourc:[1,2,7,9,10,15],source_conn_id:1,source_project_dataset_t:1,space:5,span:15,spark:[1,2,5],sparkcmd:1,spawn:3,special:[1,2,4,7],specif:[0,1,2,3,4,7,11,14,15],specifi:[0,1,2,3,4,5,11,12,14],specific_valu:1,spent:15,split:1,split_column:1,sponsor:5,spring:10,sql:[1,2,3,6,9],sql_alchemy_conn:3,sql_alchemy_conn_cmd:3,sqlachemi:1,sqlalchemi:[1,2,3],sqlite:[1,13],sqlitehook:1,sqloper:2,sqlsensor:1,squeez:1,sscursor:1,ssh:[1,12],ssh_default:1,ssh_hook:1,sshexecuteoper:1,sshhook:1,sshpass:1,ssl:1,sslcert:1,sslmode:1,stabil:5,stabl:6,stack:[6,8],stage:[1,2],stai:11,stamp:11,stand:2,standalon:2,standard:[1,2],stanza:
 3,start:[],start_dat:[0,1,2,4,11,14],stat:1,state:[0,1,2,4,7,11,14],state_d:1,statement:[1,2,7],static_babynam:1,static_babynames_partit:1,static_fold:8,static_path:1,static_url_path:8,statu:[0,1,2,3,4,5,7,11,13,14,15],stderr:0,stdout:[0,1,14],step:[1,3,14,15],steven:10,still:[1,4],sting:1,stop:[1,3],stoppag:7,storag:[1,3],store:[0,1,2,3,8,12,13],store_fil:1,store_to_xcom_kei:1,storm:5,str:[1,8,11],straightforward:13,stream:[1,5],string:[0,1,2,3,14],string_data:1,structur:[1,5,14],style:1,sub:[0,1],sub_command:1,sub_commnad:1,sub_dag:[1,2],sub_part:1,subcommand:[1,3,11,14],subdag:[],subdag_oper:2,subdagoper:2,subdir:0,subdirectori:2,subject:[1,7],sublicens:7,submiss:[7,11],submit:[1,7],subpackag:[6,12],subpartit:1,subprocess:[1,11],subscrib:[2,10],subsect:[0,11],subsequ:[0,7,11],subset:1,subtl:2,subtract:1,subtre:12,succe:[1,2],succeed:[0,1,2,4,15],success:[0,1,2,4,5,11,14],successfulli:[1,2,14],suit:5,sum:2,summit:10,sundai:11,super_us:12,supersed:7,superus:12,superuser_filt:12,sup
 pli:[1,3,12],support:[0,1,3,6,7,12],sure:[1,2,3,4,12,14],surfac:14,surgeri:5,sync:[0,1,3,11],sync_to_db:1,synchron:[1,3],syntax:7,sysconfig:3,system:[1,2,3,7],syvineckruyk:10,tab:[1,12],tabl:[1,2,8],table_exist:1,table_nam:[1,2],tablenam:1,tag:[1,4],take:[1,2,3,10,12,13,14,15],talk:[1,10,14],tardif:10,target:[1,3,11],target_field:1,target_partition_s:1,target_tim:1,task:[],task_id:[0,1,2,11,14],task_inst:[1,2],task_instance_key_str:1,task_or_task_list:1,task_param:0,task_regex:[0,1],task_stat:0,taskinst:[1,2],tbl:1,team:[1,12],tell:[0,1],templat:[],template_fold:8,template_searchpath:[1,14],templated_command:14,templates_dict:1,templates_ext:1,temporari:[1,2],tenanc:[],tenant:12,term:7,termin:[2,7],test:[],test_env:2,test_hql:1,test_mod:1,test_plugin:8,testabl:5,testview:8,text:[1,3,7,15],textfil:1,than:[1,2,5,14],thei:[0,1,2,3,5,6,8,9,11,14],them:[0,1,2,8,11,14],themselv:2,theoret:1,theori:[4,7],thereof:7,thi:[0,1,2,3,4,5,7,8,11,12,13,14,15],thin:1,thing:[1,2,4,14],think:[5,14],thi
 rd:7,this_dag_will_be_discov:2,those:[1,2,3,7,12],though:[1,2,5],thousand:2,thread:1,three:[2,3],threshold:8,thrift:1,through:[1,2,6,7,11,13,14],throughout:1,thu:1,ticket:[0,12],tighten:12,time:[1,2,3,4,8,11,12,14,15],timedelta:[1,2,4,11,14],timedeltasensor:[1,4],timefram:1,timeout:[0,1,2],timesensor:1,timestamp:[1,2,11],tip:[2,10],titl:[2,7],tls_ca_cert:1,tls_client_cert:1,tls_client_kei:1,tls_hostnam:1,tls_ssl_version:1,tmp:[1,2],tmp_dir:1,tmp_file:1,todai:1,togeth:[1,2],token:1,toler:[1,2],tomorrow:1,tomorrow_d:1,tomorrow_ds_nodash:1,too:[1,2,11],tool:[1,2,6,8],toolbox:8,top:[],topic:3,tornado:0,tort:7,tour:[13,14],track:[2,7,14],tracker:1,trade:7,trademark:7,tradit:1,tradition:2,train:7,transact:1,transfer:[1,2,7],transform:7,translat:[1,7],transpar:15,travers:1,treat:[1,2],tree:[],tree_view:1,tri:[1,12],trick:10,trigger:[],trigger_dag:[0,11],trigger_dag_id:1,trigger_rul:[1,2],triggerdagrunoper:[1,10],triggerrul:1,troubl:6,troubleshoot:[5,15],truncat:1,truth:1,ts_dim:1,ts_nodash
 :1,tsv:1,tty:1,tunnel:[1,12],tupl:1,turn:[1,2,12],tutori:[],twice:1,two:[1,2,14],txt:1,type:[0,1,2,4,7,9,12],typic:[1,2],ubuntu:3,udf_config:1,uid:12,ultim:[1,14],unarchiv:1,unassign:2,unavail:3,unblock:11,undefin:12,under:[1,2,3,7,8,10],undergo:5,underli:8,underscor:3,understand:[1,4,8,14,15],undesir:11,unexpect:2,unexpectedli:2,union:7,uniqu:[1,14],unit:[3,5,12],unittest:1,unix:1,unless:[1,4,7],unpack:[1,2],unpaus:0,unset:1,unsound:[1,2],unsuccess:1,unsupport:1,until:[1,2,3,5],up_for_retri:1,updat:[1,2,4,12],updateonli:1,upgrad:0,upgradedb:0,upload:1,upon:[2,3,13],upstream:[0,1,2,4,11],upstream_fail:[1,2],upstream_list:1,upstream_task_id:1,uri:[1,2,3,12],url:[1,2,3,8,9,12],usag:[0,1,14],use_beelin:[1,12],user:[0,1,2,3,5,6,12,14],user_defined_macro:1,user_filt:12,user_name_attr:12,user_program_argu:1,usernam:[1,3,12],usr:3,usual:[2,3],utf8:1,utf:1,util:[1,5,13],uuid:1,val:[0,1],valid:[],valu:[0,1,2,3,4,12,14,15],valuecheckoper:1,variabl:[],vast:1,verbal:7,verbos:[1,14],veri:[0,1,2,
 4,10,13,14],verifi:[2,4],version:[0,1,2,5,6,7],vertia:1,vertica:[1,6],vertica_conn_id:1,vertica_default:1,verticahook:1,verticaoper:1,verticaql:1,verticatohivetransf:1,via:[1,2,12],video:[1,10],view:[],virtual:1,virtualenv:2,visibl:[1,2],visual:[5,8,9,14,15],vocabulari:2,volum:1,wai:[1,2,3,4,6,8,11,12,14,15],wait:[0,1,2],wait_for_downstream:[1,4],wait_for_empty_queu:2,walk:[1,14],want:[1,2,3,4,8,11,12,14],warranti:7,wasn:[1,4],wasnt:[],watch:[1,3,4],weapon:9,web:[],webhdfs_conn_id:1,webhdfs_default:1,webhdfshook:1,webhdfssensor:1,webserv:[0,3,4,12,13,14],week:11,weekli:11,weight:1,well:[1,2,3,4,11],were:1,what:[],whatev:[1,2,3,6],when:[1,2,3,4,5,8,9,11,12,14],whenev:1,where:[1,2,3,4,7,11,12,14,15],where_claus:1,wherev:[1,7],whether:[1,7,11],which:[0,1,2,3,4,7,11,12,13,14],whitelist:12,who:8,whole:[1,2,7],whom:7,whose:11,why:[],wiedmer:10,wiki:10,wildcard:1,wildcard_kei:1,wildcard_match:1,win:1,window:11,winter:10,wire:2,within:[0,1,2,3,4,7],without:[0,1,2,7,11,12],won:[1,4,6],word:[
 2,3,11,15],work:[0,1,2,3,4,5,7,8,9,11,12,13],worker:[0,1,2,3,4,5,6,8,12,14],worker_timeout:0,workerclass:0,workflow:[],workload:8,worldwid:7,worri:14,worth:1,would:[1,2,3,4,5,11,12,14],wouldn:[1,4],wrap:[1,14],wrapper:1,write:[3,5,7,8,9,14],write_disposit:1,write_empti:1,written:[1,7,14],wrong:14,www:[1,2,7],x01:1,xcom:[],xcom_al:1,xcom_pul:[1,2],xcom_push:[1,2],xcom_return_kei:1,xml:12,xst:12,xxx:1,yaml:2,year:[1,11],yearli:11,yellow:1,yesterdai:1,yesterday_d:1,yesterday_ds_nodash:1,yet:[1,2,5,12,15],you:[1,2,3,4,5,6,7,8,9,11,12,13,14,15],your:[],yourself:1,youtub:1,yum:6,yvinec:10,yyyi:[0,1,2],yyyymmdd:1,zero:1,zip:[1,2],zip_dag:2,zip_dag_cont:2,zoom:2},titles:["Command Line Interface","API Reference","Concepts","Configuration","FAQ","Apache Airflow (incubating) Documentation","Installation","License","Plugins","Data Profiling","Project","Scheduling & Triggers","Security","Quick Start","Tutorial","UI / Screenshots"],titleterms:{"default":[1,2,14],"function":2,"import":14,addit
 :2,adhoc:9,airflow:[5,6,8],apach:5,api:1,argument:[2,14],assign:2,authent:12,backend:3,backfil:14,baseoper:1,basesensoroper:1,beyond:5,bitshift:2,branch:2,build:8,celeri:3,chart:[9,15],cluster:2,code:15,command:[0,14],committ:10,commun:[1,3],composit:2,concept:2,configur:3,connect:[2,3],content:5,context:[2,15],contribut:[1,3],core:2,dag:[2,11,14,15],data:9,definit:14,depend:14,disclaim:[],document:[2,5],durat:15,enabl:12,enterpris:12,exampl:[8,14],executor:1,extern:11,extra:6,faq:4,file:14,form:9,gantt:15,get:6,ghe:12,github:12,graph:15,histori:10,hook:[1,2],horizon:5,idea:2,incub:5,instal:6,instanc:[2,15],instanti:14,integr:3,interfac:[0,8],jinja:[2,14],keep:11,kerbero:12,ldap:12,licens:7,limit:12,line:[0,14],link:10,log:3,macro:1,manag:2,menu:15,meso:3,metadata:14,mind:11,model:1,modul:14,multi:12,next:[13,14],note:2,oper:[1,2],option:3,out:3,own:12,packag:[2,6],password:12,pipelin:14,plugin:8,polici:2,pool:2,principl:5,profil:9,project:10,queri:9,queue:2,quick:13,recap:14,refer:
 1,resourc:10,roadmap:10,roll:12,rule:2,run:[11,14],scale:3,schedul:11,scope:2,screenshot:[9,15],script:14,secur:12,set:[3,12,14],sla:2,start:13,subdag:2,systemd:3,task:[2,14,15],templat:[2,14],tenanc:12,test:14,top:8,tree:15,trigger:[2,11],tutori:14,undead:2,upstart:3,valid:14,variabl:[1,2,15],view:15,web:12,what:[8,13,14],why:8,workflow:2,xcom:2,your:12,zombi:2}})
\ No newline at end of file
+Search.setIndex({envversion:47,filenames:["cli","code","concepts","configuration","faq","index","installation","license","plugins","profiling","project","scheduler","security","start","tutorial","ui"],objects:{"airflow.contrib":{hooks:[1,1,0,"-"],operators:[1,1,0,"-"]},"airflow.contrib.executors.mesos_executor":{MesosExecutor:[1,0,1,""]},"airflow.contrib.hooks":{BigQueryHook:[1,0,1,""],CloudantHook:[1,0,1,""],FTPHook:[1,0,1,""],GoogleCloudStorageHook:[1,0,1,""],SSHHook:[1,0,1,""],VerticaHook:[1,0,1,""]},"airflow.contrib.hooks.BigQueryHook":{get_conn:[1,2,1,""],get_pandas_df:[1,2,1,""],get_service:[1,2,1,""],insert_rows:[1,2,1,""]},"airflow.contrib.hooks.CloudantHook":{db:[1,2,1,""]},"airflow.contrib.hooks.FTPHook":{close_conn:[1,2,1,""],create_directory:[1,2,1,""],delete_directory:[1,2,1,""],delete_file:[1,2,1,""],describe_directory:[1,2,1,""],get_conn:[1,2,1,""],list_directory:[1,2,1,""],rename:[1,2,1,""],retrieve_file:[1,2,1,""],store_file:[1,2,1,""]},"airflow.contrib.hooks.Google
 CloudStorageHook":{download:[1,2,1,""],get_conn:[1,2,1,""],upload:[1,2,1,""]},"airflow.contrib.hooks.SSHHook":{Popen:[1,2,1,""],check_output:[1,2,1,""],tunnel:[1,2,1,""]},"airflow.contrib.hooks.VerticaHook":{get_conn:[1,2,1,""]},"airflow.contrib.hooks.gcs_hook":{GoogleCloudStorageHook:[1,0,1,""]},"airflow.contrib.operators":{QuboleOperator:[1,0,1,""],SSHExecuteOperator:[1,0,1,""],VerticaOperator:[1,0,1,""],VerticaToHiveTransfer:[1,0,1,""]},"airflow.contrib.operators.bigquery_operator":{BigQueryOperator:[1,0,1,""]},"airflow.contrib.operators.bigquery_to_gcs":{BigQueryToCloudStorageOperator:[1,0,1,""]},"airflow.contrib.operators.gcs_download_operator":{GoogleCloudStorageDownloadOperator:[1,0,1,""]},"airflow.contrib.operators.hipchat_operator":{HipChatAPIOperator:[1,0,1,""],HipChatAPISendRoomNotificationOperator:[1,0,1,""]},"airflow.executors":{CeleryExecutor:[1,0,1,""],LocalExecutor:[1,0,1,""],SequentialExecutor:[1,0,1,""]},"airflow.hooks":{DbApiHook:[1,0,1,""],DruidHook:[1,0,1,""],Hi
 veCliHook:[1,0,1,""],HiveMetastoreHook:[1,0,1,""],HiveServer2Hook:[1,0,1,""],HttpHook:[1,0,1,""],MsSqlHook:[1,0,1,""],MySqlHook:[1,0,1,""],PostgresHook:[1,0,1,""],PrestoHook:[1,0,1,""],S3Hook:[1,0,1,""],SqliteHook:[1,0,1,""],WebHDFSHook:[1,0,1,""]},"airflow.hooks.DbApiHook":{bulk_dump:[1,2,1,""],bulk_load:[1,2,1,""],get_conn:[1,2,1,""],get_cursor:[1,2,1,""],get_first:[1,2,1,""],get_pandas_df:[1,2,1,""],get_records:[1,2,1,""],insert_rows:[1,2,1,""],run:[1,2,1,""]},"airflow.hooks.DruidHook":{construct_ingest_query:[1,2,1,""],get_conn:[1,2,1,""],load_from_hdfs:[1,2,1,""]},"airflow.hooks.HiveCliHook":{load_file:[1,2,1,""],run_cli:[1,2,1,""],test_hql:[1,2,1,""]},"airflow.hooks.HiveMetastoreHook":{check_for_named_partition:[1,2,1,""],check_for_partition:[1,2,1,""],get_databases:[1,2,1,""],get_metastore_client:[1,2,1,""],get_partitions:[1,2,1,""],get_table:[1,2,1,""],get_tables:[1,2,1,""],max_partition:[1,2,1,""],table_exists:[1,2,1,""]},"airflow.hooks.HiveServer2Hook":{get_pandas_df:[1,2,
 1,""],get_records:[1,2,1,""]},"airflow.hooks.HttpHook":{get_conn:[1,2,1,""],run:[1,2,1,""],run_and_check:[1,2,1,""]},"airflow.hooks.MsSqlHook":{get_conn:[1,2,1,""]},"airflow.hooks.MySqlHook":{bulk_load:[1,2,1,""],get_conn:[1,2,1,""]},"airflow.hooks.PrestoHook":{get_conn:[1,2,1,""],get_first:[1,2,1,""],get_pandas_df:[1,2,1,""],get_records:[1,2,1,""],run:[1,2,1,""]},"airflow.hooks.S3Hook":{check_for_bucket:[1,2,1,""],check_for_key:[1,2,1,""],check_for_prefix:[1,2,1,""],check_for_wildcard_key:[1,2,1,""],get_bucket:[1,2,1,""],get_conn:[1,2,1,""],get_key:[1,2,1,""],get_wildcard_key:[1,2,1,""],list_keys:[1,2,1,""],list_prefixes:[1,2,1,""],load_file:[1,2,1,""],load_string:[1,2,1,""]},"airflow.hooks.SqliteHook":{get_conn:[1,2,1,""]},"airflow.hooks.WebHDFSHook":{check_for_path:[1,2,1,""],get_conn:[1,2,1,""],load_file:[1,2,1,""]},"airflow.macros":{ds_add:[1,3,1,""],ds_format:[1,3,1,""],hive:[1,1,0,"-"],random:[1,3,1,""]},"airflow.macros.hive":{closest_ds_partition:[1,3,1,""],max_partition:[1,
 3,1,""]},"airflow.models":{BaseOperator:[1,0,1,""],Connection:[1,0,1,""],DAG:[1,0,1,""],DagBag:[1,0,1,""],TaskInstance:[1,0,1,""]},"airflow.models.BaseOperator":{clear:[1,2,1,""],dag:[1,4,1,""],detect_downstream_cycle:[1,2,1,""],downstream_list:[1,4,1,""],execute:[1,2,1,""],get_direct_relatives:[1,2,1,""],get_flat_relatives:[1,2,1,""],get_task_instances:[1,2,1,""],has_dag:[1,2,1,""],on_kill:[1,2,1,""],post_execute:[1,2,1,""],pre_execute:[1,2,1,""],prepare_template:[1,2,1,""],render_template:[1,2,1,""],render_template_from_field:[1,2,1,""],run:[1,2,1,""],schedule_interval:[1,4,1,""],set_downstream:[1,2,1,""],set_upstream:[1,2,1,""],upstream_list:[1,4,1,""],xcom_pull:[1,2,1,""],xcom_push:[1,2,1,""]},"airflow.models.Connection":{extra_dejson:[1,4,1,""]},"airflow.models.DAG":{add_task:[1,2,1,""],add_tasks:[1,2,1,""],clear:[1,2,1,""],cli:[1,2,1,""],concurrency_reached:[1,4,1,""],crawl_for_tasks:[1,2,1,""],create_dagrun:[1,2,1,""],deactivate_stale_dags:[1,5,1,""],deactivate_unknown_dags:[
 1,5,1,""],filepath:[1,4,1,""],folder:[1,4,1,""],get_template_env:[1,2,1,""],is_paused:[1,4,1,""],latest_execution_date:[1,4,1,""],normalize_schedule:[1,2,1,""],run:[1,2,1,""],set_dependency:[1,2,1,""],sub_dag:[1,2,1,""],subdags:[1,4,1,""],sync_to_db:[1,5,1,""],tree_view:[1,2,1,""]},"airflow.models.DagBag":{bag_dag:[1,2,1,""],collect_dags:[1,2,1,""],dagbag_report:[1,2,1,""],get_dag:[1,2,1,""],kill_zombies:[1,2,1,""],process_file:[1,2,1,""],size:[1,2,1,""]},"airflow.models.TaskInstance":{are_dependencies_met:[1,2,1,""],are_dependents_done:[1,2,1,""],clear_xcom_data:[1,2,1,""],command:[1,2,1,""],current_state:[1,2,1,""],error:[1,2,1,""],evaluate_trigger_rule:[1,2,1,""],generate_command:[1,5,1,""],is_premature:[1,2,1,""],is_queueable:[1,2,1,""],is_runnable:[1,2,1,""],key:[1,4,1,""],next_retry_datetime:[1,2,1,""],pool_full:[1,2,1,""],ready_for_retry:[1,2,1,""],refresh_from_db:[1,2,1,""],run:[1,2,1,""],xcom_pull:[1,2,1,""],xcom_push:[1,2,1,""]},"airflow.operators":{BashOperator:[1,0,1,""]
 ,BranchPythonOperator:[1,0,1,""],DummyOperator:[1,0,1,""],EmailOperator:[1,0,1,""],ExternalTaskSensor:[1,0,1,""],GenericTransfer:[1,0,1,""],HdfsSensor:[1,0,1,""],Hive2SambaOperator:[1,0,1,""],HiveOperator:[1,0,1,""],HivePartitionSensor:[1,0,1,""],HiveToDruidTransfer:[1,0,1,""],HiveToMySqlTransfer:[1,0,1,""],HttpSensor:[1,0,1,""],MetastorePartitionSensor:[1,0,1,""],MsSqlOperator:[1,0,1,""],MsSqlToHiveTransfer:[1,0,1,""],MySqlOperator:[1,0,1,""],MySqlToHiveTransfer:[1,0,1,""],NamedHivePartitionSensor:[1,0,1,""],PostgresOperator:[1,0,1,""],PrestoCheckOperator:[1,0,1,""],PrestoIntervalCheckOperator:[1,0,1,""],PrestoValueCheckOperator:[1,0,1,""],PythonOperator:[1,0,1,""],S3KeySensor:[1,0,1,""],S3ToHiveTransfer:[1,0,1,""],ShortCircuitOperator:[1,0,1,""],SimpleHttpOperator:[1,0,1,""],SlackAPIOperator:[1,0,1,""],SlackAPIPostOperator:[1,0,1,""],SqlSensor:[1,0,1,""],TimeSensor:[1,0,1,""],TriggerDagRunOperator:[1,0,1,""],WebHdfsSensor:[1,0,1,""]},"airflow.operators.BashOperator":{execute:[1,2,
 1,""]},"airflow.operators.SlackAPIOperator":{construct_api_call_params:[1,2,1,""],execute:[1,2,1,""]},"airflow.operators.docker_operator":{DockerOperator:[1,0,1,""]},"airflow.operators.sensors":{BaseSensorOperator:[1,0,1,""]},airflow:{executors:[1,1,0,"-"],hooks:[1,1,0,"-"],macros:[1,1,0,"-"],models:[1,1,0,"-"],operators:[1,1,0,"-"]}},objnames:{"0":["py","class","Python class"],"1":["py","module","Python module"],"2":["py","method","Python method"],"3":["py","function","Python function"],"4":["py","attribute","Python attribute"],"5":["py","staticmethod","Python static method"]},objtypes:{"0":"py:class","1":"py:module","2":"py:method","3":"py:function","4":"py:attribute","5":"py:staticmethod"},terms:{"00am":1,"01t23":11,"10gb":1,"10pm":2,"128m":1,"2am":1,"5gb":1,"__class__":2,"__doc__":2,"__init__":[1,2],"__name__":[2,8],"_cmd":3,"_host":12,"abstract":[1,2,5,8],"boolean":[1,2],"byte":[1,4],"case":[1,2,4,11,12,14],"char":1,"class":[0,1,2,6,7,8,14],"default":[],"export":[0,1,13],"final
 ":1,"float":1,"function":[],"import":[],"int":1,"long":1,"new":[1,2,4,8,11,12],"return":[0,1,2,4,8],"short":1,"static":[1,5,8],"super":[1,12],"switch":[3,12],"true":[1,2,4,12,14],"try":[1,2,3],"var":1,"while":[1,2,4,5,7,11,13,14,15],abbrevi:2,abil:2,abl:[1,3,4,12,13,14],about:[1,2,3,7,10,11,14],abov:[2,3,7,12,15],absenc:2,absolut:2,accept:[1,5,7],access:[0,1,2,3,8,12],access_logfil:0,access_token:15,accord:12,accordingli:2,account:[1,12],acquir:1,across:[3,15],act:[1,2,7,14],action:[1,15],activ:[1,2,4,8,11,12,14],active_dag_id:1,actual:[1,2,4,14],acycl:[1,2,5],add:[1,2,3,4,7,12],add_task:1,addendum:7,addit:[],addition:12,addprinc:12,adhoc:[],adjust:[3,12],admin:[2,3,4,8,9,12,13],admin_view:8,advanc:[1,14],advantag:12,advis:[1,7],affect:2,after:[],afterward:1,against:[1,2,4,7,12,14],agari:10,aggregate_db_message_job:2,aggregate_db_message_job_cmd:2,ago:4,agre:7,agreement:[2,7],airbnb:[1,2,10,14],airflow:[],airflow__:3,airflow__core__sql_alchemy_conn:3,airflow_config:3,airflow_conn_:[
 2,3],airflow_conn_postgres_mast:2,airflow_hom:[3,4,8,13],airflow_set:2,airflow_tmp_dir:1,airflowignor:1,airflowplugin:8,airflowtestplugin:8,airlfow:[],aiyzngq:1,alchemi:3,alert:[1,2,8],algorithm:1,align:4,all:[],all_db:6,all_don:[1,2],all_fail:[1,2],all_success:[1,2],alleg:7,alloc:1,allow:[0,1,2,3,4,5,8,9,11,12,13,14,15],allow_large_result:1,allowed_st:1,allowed_team:12,allowinsert:1,alon:7,along:[1,4,7,14],alongsid:[1,7,11],alreadi:1,alright:14,also:[1,2,3,4,6,7,8,11,12,14],alter:[1,2,3,4,12],altern:[1,11],altert:1,although:12,alwai:[1,2,15],amazon:3,amount:[1,2],analys:15,analyz:2,anand:10,anchor:1,ani:[1,2,3,4,7,14,15],annot:7,announc:10,anomali:8,anoth:[],ansibl:3,anyon:12,anyth:[2,14],anytim:2,anywai:1,anywher:[1,2],aoen:10,apach:[],api:[],api_call_param:1,api_kei:15,api_param:1,api_vers:1,apikei:15,apiv2:1,app:1,app_id:1,appar:1,appear:[1,2,7],append:[1,3],appendix:7,appli:[1,2,6,7,11],applic:[1,7,8,9,12],appropri:[2,7,12],arbitrari:[2,5],architectur:5,archiv:[1,7],archive1:1,
 archive2:1,are_dependencies_met:1,are_dependents_don:1,aren:[1,2],arg:1,argument:[],aris:7,around:[1,5,14],arrai:[1,5,8],arthur:10,artwr:10,ascii:1,asf:5,assert:7,asset:1,assign:[],associ:[1,2,7,11],assum:[1,2,3,7,14],asterisk:12,async:6,atom:2,attach:[1,2,7],attach_to:1,attack:12,attempt:[0,1],attent:[1,4],attr:1,attribut:[0,1,2,7],audit:8,auth:[1,12],auth_backend:12,auth_token:1,authen:12,authent:[],authmechan:1,author:[1,3,5,7,12,14,15],authorship:7,auto:4,autocommit:1,automat:[1,2,3],avail:[1,2,3,7,8,9],averag:1,avoid:2,awai:1,awar:2,azkaban:5,back:[1,4,8],backend:[],backfil:[],background:[1,14],backoff:1,backup:3,bag:1,bag_dag:1,balanc:2,bar:[1,2],bare:14,base64:4,base:[],base_dag:1,base_executor:[1,8],base_hook:[1,8],base_url:1,basedag:1,basedagbag:1,basedn:12,baseexecutor:[1,8],basehook:[1,2,8],baseview:8,bash:[1,2,14],bash_1:2,bash_command:[1,2,14],bash_command_to_run:3,bash_oper:[1,14],bash_queu:14,bashoper:[1,2,14],basi:7,basic:[2,4,6,8,9,14],bcc:1,bcrypt:12,beauchemin:10,
 becaus:[1,2],becom:[1,2,5,8,14],beelin:1,been:[1,2,3,7,11,12],befor:[0,1,2,4,12],begin:[2,11],behalf:7,behav:[1,4],behavior:[1,2],behind:[1,3,6,11],bellow:4,belong:[1,12],below:[1,7,8,13,14],benefici:7,best:[1,10],better:[4,14],between:[1,2,14],beyond:[],bigqueri:1,bigquery_conn_id:1,bigquery_default:1,bigquery_oper:1,bigquery_to_gc:1,bigqueryhook:1,bigqueryoper:1,bigquerytocloudstorageoper:1,bin:2,bind:7,bind_password:12,bind_us:12,bit:3,blob:[1,14],block:[1,2,14,15],blog:[7,10],blueprint:8,bodi:1,boilerpl:7,bolk:10,bolkedebruin:10,bonu:1,bool:1,boot:3,both:[1,4],bother:14,boto:1,bottleneck:15,boundary_queri:1,box:[1,3,13],bql:1,bracket:[7,14],branch:[],branch_fals:2,branchpythonoper:[1,2],broker:[0,3],broker_api:0,broker_url:3,brought:10,brows:[2,4],bruin:10,bucket:[1,3],bucket_kei:1,bucket_nam:1,buffer:1,bug:3,build:[],built:[1,2,3,5,8,9,14],bulk:[4,15],bulk_dump:1,bulk_load:1,bump:2,bundl:1,burn:0,but_this_dag_will_not:2,c12318391:1,cacert:12,cach:12,call:[1,2,3,4,14],callabl:1,
 callback:12,can:[],cannot:[1,2,7,14],capabl:8,capac:[1,2],capacityschedul:1,card:1,care:[0,3],carri:[2,7],cast:1,cat:1,categori:8,caus:[4,7],celeri:[],celery_result_backend:3,celeryexecutor:[1,2,3,4,6,11],cell:1,central:2,cert:1,certain:[1,2],certif:1,cfg:[1,2,3,4,11,12,13,14],chain:2,chang:[3,5,7,11,12,13,14],channel:[1,10],charact:7,character:2,charg:[4,7],charset:1,chart:[],chat:10,check:[0,1,2,4],check_for_bucket:1,check_for_kei:1,check_for_named_partit:1,check_for_partit:1,check_for_path:1,check_for_prefix:1,check_for_wildcard_kei:1,check_head:1,check_oper:1,check_output:1,checkoper:1,chef:[3,10],child:[1,2],child_dag:2,child_dag_nam:2,chmod:12,choic:[0,1,14],choos:[1,2,7],chri:10,circuit:1,claim:7,clariti:5,claus:1,clean:1,cleanup:[1,4],clear:[0,1,2,4,11,15],clear_xcom_data:1,cli:[1,2,3,4,11],click:[4,12,14,15],client:[1,6,12],client_id:12,client_secret:12,close:[1,4,12],close_conn:1,closest:1,closest_ds_partit:1,cloud:[1,3,6],cloudant:[1,6],cloudant_conn_id:1,cloudant_default
 :1,cloudanthook:1,cluster:[],cluster_label:1,cmd:1,cmdline:1,code:[],col:1,collabor:5,collect:[1,2,8],collect_dag:1,color:1,column:1,com:[1,2,12,14],combin:[2,7],come:[1,11],comma:[0,1,2],command_typ:1,comment:7,commerci:7,commit:[1,10,12],commit_everi:1,committ:[],common:[2,3,4,7,10,14],commonli:1,compani:8,compar:5,comparison:1,compil:[4,7],complet:[1,2,3,5],complex:[1,2,5,8],compli:7,complianc:7,complic:14,compon:8,compos:[2,12,14],comprehens:15,compress:1,comput:[1,7],conceptu:2,concern:2,concur:4,concurr:[0,1,4],concurrency_reach:1,condit:[1,2,6,7],conf:[0,1,3],config:[8,12],configur:[],confirm:[0,4,11],confirm_prompt:1,confus:[1,2,4],conjunct:[0,2,13],conn_id:[1,2,3],conn_typ:1,connect:[],connect_timeout:1,connecton:9,consequenti:7,consid:[1,2,3],consider:1,consist:[1,2,5,7],conspicu:7,constant:[1,2],constitu:2,constitut:7,constraint:1,constru:7,construct:1,construct_api_call_param:1,construct_ingest_queri:1,constructor:[1,14],contact:[3,7],contain:[1,2,3,4,7,11,12,14,15],cont
 ent:[],context:[],continu:[1,5],contract:7,contrib:[1,2,12],contributor:[7,10],contributori:7,control:[1,2,7,15],conveni:[1,2],convent:2,convers:7,convert:1,cool:1,coordin:1,copi:[1,3,7,12],copyright:[7,12],core:[],correct:2,correpsond:2,correspond:[1,2,3,4],could:[1,2,14],count:1,counterclaim:7,cours:12,cover:11,cpu:[1,8],craft:1,crawl:[1,11],crawl_for_task:1,creat:[],create_dagrun:1,create_directori:1,creation:1,credenti:12,credit:12,criccomini:10,criteria:[1,2],criterion:1,critic:[1,2],cron:[1,4,11],cross:[2,7,14],crt:12,crypto:[],csv:1,curli:14,current:[1,15],current_st:1,cursor:1,custom:8,customari:7,cycl:[1,2,14],daemon:[0,1,3],dag:[],dag_1:2,dag_2:2,dag_fold:[1,2],dag_id:[0,1,2,4,14],dag_md:2,dag_run:1,dag_run_obj:1,dag_stat:0,dagbag:[0,1,4],dagbag_report:1,dagfil:2,dagrun:[0,1,4],dagrun_timeout:1,dags_fold:[3,4],dai:[1,11,13,14],daili:[1,4,11],damag:7,dan:[1,10],data:[],data_profiler_filt:12,databas:[0,1,2,3,5,6,8,9,13,14],datafram:1,dataset:1,datasourc:1,date:[0,1,2,7,11,14
 ],date_filter_column:1,datestamp:1,datetim:[1,2,4,11,14],dateutil:1,davydov:10,days_back:1,db_tabl:1,db_tap_id:1,db_update_kei:1,db_update_mod:1,dbapi_hook:1,dbapihook:1,dbexportcmd:1,dbimportcmd:1,dbtap_id:1,dbtapquerycmd:1,deactiv:1,deactivate_stale_dag:1,deactivate_unknown_dag:1,debug:[0,1,14],decid:[1,3],decis:5,declar:1,deep:1,def:[1,2,8],default_arg:[1,2,4,14],default_login:12,default_queu:2,defend:7,defer:2,deferred_op:2,defin:[1,2,3,4,5,7,8,11,12,14],definit:[],del:1,delai:[1,4],delegate_to:1,delet:[0,1,2,11,15],delete_directori:1,delete_fil:1,deliber:[1,7],delimit:[0,1,2],demonstr:2,dep:1,depend:[],depends_on_past:[0,1,2,4,14],deploi:8,deploy:8,depth:14,deriv:[1,3,7,8],descend:2,describ:[1,2,4,7],describe_directori:1,descript:[1,2,7],deseri:[0,1,2],deserialize_json:2,design:[2,7,11],desir:[1,14],destin:1,destination_cloud_storage_uri:1,destination_conn_id:1,destination_dataset_t:1,destination_filepath:1,destination_t:1,detail:[1,2,4,12,15],detect:[1,8],detect_downstream_cyc
 l:1,determin:[1,2,7],dev:[3,6,10],devel:6,devel_hadoop:6,develop:[0,1,12,14],deviat:1,dialog:11,dict:[0,1,2],dictionari:[1,2,4,14],did:14,die:2,differ:[1,2,7,8,14,15],dimens:1,direct:[1,2,3,5,7],directli:[1,2,4,9],directori:[0,1,2,3,12,14],dirti:13,disabl:1,disclaim:[5,7],discuss:7,disk:1,displai:[3,7,11,14],disregard:[1,14],distinct:1,distribut:[1,2,6,7],do_pickl:0,doc:[1,2,4,14],doc_json:2,doc_md:2,doc_rst:2,doc_yaml:2,docker:[1,10],docker_oper:1,docker_url:1,dockeroper:[1,2],document:[],doe:[0,1,2,3,4,5,6,7,12,14],does_not_exist:1,doesn:[1,2,14],domain:12,don:[1,2,3,6,7,11],done:[1,2,4,11,12,14],donot_pickl:[0,1],door:2,dot:[1,2],doubl:[1,3,14],down:[0,2],download:[1,3],downstream:[0,1,2,11],downstream_list:1,downstream_task_id:1,drive:3,driven:[1,8],driver:3,drop:[1,8],druid:[1,6],druid_datasourc:1,druid_ingest_conn_id:1,druid_ingest_default:1,druid_query_conn_id:1,druid_query_default:1,druidhook:1,dry:0,dry_run:[0,1],ds_add:[1,14],ds_format:1,ds_nodash:1,dttm:1,dubiou:1,dummi:[
 1,2,8],dummy_1:2,dummy_oper:2,dummy_task:2,dummyoper:[1,2],dump:1,durat:[],dure:[3,15],dynam:[],each:[1,2,4,7,10,11,14],earlier:14,easi:[2,3,5,8,9,12,15],easier:[1,7],easiest:6,easili:[1,2,4,5,9,14],echo:[2,14],ecosystem:8,edit:[3,15],editori:7,effect:[1,2],effici:1,effort:5,egg:3,either:[1,2,3,7,12,13],elabor:7,elaps:1,elast:1,electron:7,eleg:5,element:3,els:13,elsewher:2,email:[1,2,10,12,14],email_on_failur:[1,14],email_on_retri:[1,14],emailoper:[1,2],emit:[2,4,14],empti:[1,2],enabl:[],enclos:7,encod:[1,4],encrypt:[],encrypt_s3_log:3,end:[1,7,11],end_dat:[0,1,14],endors:5,endpoint:1,enforc:[2,4],engin:[1,4,5],ensur:4,enterpris:[],entir:[0,4],entiti:7,entri:[1,11],env:[1,2],environ:[1,2,3,5,6,9,11,14,15],ep_data_pipeline_db_msg_agg:2,equival:[2,6,14],era:4,error:[0,1],error_logfil:0,especi:[2,4,14],essenti:1,etc:[1,2,3,12],eval:1,evalu:[1,2,4,11,14],evaluate_trigger_rul:1,even:[1,2,7,9],event:[2,7,14],eventlet:0,ever:1,everi:[1,2,11,12,14],everyon:14,everyth:[14,15],exactli:[1,15],
 exampl:[],example1:13,example_bash_oper:13,example_dag:[2,14],example_team_1:12,example_team_2:12,exce:1,except:[1,2,4,7,14],exchang:[2,5],exclud:[0,7],exclude_subdag:0,exclus:[7,11],execut:[0,1,2,3,4,5,7,11,14],execution_d:[0,1,2,14],execution_date_fn:1,execution_delta:1,execution_timeout:[1,2],exercis:7,exhaust:3,exist:[0,1,2,3,4,14],exit:[0,12],expect:[1,2,4,5],expir:1,expiration_d:1,explain:1,explan:14,explicit:5,explicit_op:2,explicitli:[1,2,7,12,14],exponenti:1,export_dir:1,export_format:1,expos:[1,2,3,8,12],express:[1,4,7,11],ext:1,extend:5,extens:[1,5],extern:[],external_dag_id:1,external_task_id:1,external_trigg:1,externaltasksensor:1,extra:[],extra_dejson:1,extra_opt:1,extract:1,extract_queri:1,fact:2,factori:2,fail:[0,1,2,3,4,11,14,15],failur:[],fair:1,fairli:13,fall:10,fals:[0,1,3,11,14],familiar:[2,14],faq:[],fast:13,fault:[1,2],featur:[1,2,3,6,8,14,15],feb:12,fee:7,fernet:4,fernet_kei:4,fetch:[1,3],few:[1,3,9,13,14],field:[1,7],field_delimit:1,field_dict:1,fields_termi
 nated_bi:1,fifti:7,figur:[1,2,11],file1:1,file2:1,file:[],file_path:1,filenam:[1,2],filepath:[0,1],filestatu:1,filesystem:3,fill:[1,2,9,12],filter:[0,1,2,12],filter_by_own:12,find:[1,3,4,14,15],fine:14,finish:1,fire:[1,2,3,4],first:[0,1,2,3,4,10,11,13,14],fit:[1,5,7],fix:[1,11],flag:[1,11],flag_upstream_fail:1,flask:[0,8,9],flask_admin:8,flask_blueprint:8,flask_login:12,flat:1,flavor:8,flexibl:[1,14],flower:[0,3],flower_conf:0,folder:[1,3,8,11,13,14],follow:[1,2,3,5,7,12,14],foo:[1,2],foo_:4,foo_dag_id:4,foobar:3,forc:[0,1],force_pul:1,foreground:0,foreign:1,form:[],format:[1,2,3,4,7],forward:[1,4],found:[1,3,12],foundat:[5,7,10,13],four:2,framework:[3,8],free:[2,7],frm:1,from:[0,1,2,3,4,5,7,8,9,10,11,12,13,14,15],from_nam:1,ftp:1,ftp_conn_id:1,ftp_default:1,ftphook:1,full:[1,10],full_filepath:1,fulli:[1,4,5,12],functiontyp:1,fundament:14,further:[5,13,14],futur:[1,11],galaxi:8,gantt:[],garag:2,gate:12,gcp_api:6,gcp_api_base_hook:1,gcs_download_oper:1,gcs_hook:1,gender:1,gener:[0,1,
 2,5,7,8,12,14,15],generate_command:1,generictransf:1,get:[],get_bucket:1,get_conn:1,get_connect:2,get_cursor:1,get_dag:1,get_databas:1,get_direct_rel:1,get_first:1,get_flat_rel:1,get_kei:1,get_metastore_cli:1,get_pandas_df:1,get_partit:1,get_partitions_by_filt:1,get_partitions_by_nam:1,get_record:1,get_servic:1,get_tabl:1,get_task_inst:1,get_template_context:1,get_template_env:1,get_wildcard_kei:1,gevent:0,ghe:[],ghe_oauth:12,ghost:1,git:3,github:[],github_enterpris:12,github_enterprise_auth:12,githubusercont:1,gitignor:1,gitter:10,give:[2,7],given:[1,2],glanc:15,global:[1,2,4],goal:14,goe:[1,4,14],good:2,goodby:2,goodwil:7,googl:[1,3,6],google_cloud_storage_conn_id:1,google_cloud_storage_default:1,googlecloudbasehook:1,googlecloudstoragedownloadoper:1,googlecloudstoragehook:1,govern:7,grab:1,grade:3,grai:1,grant:7,graph:[],great:[2,3],greater:1,green:1,grossli:7,group:[1,3,12],gunicorn:[0,6],hack:1,had:[1,4],hadoop:[1,6,10,12],hadoop_dependency_coordin:1,hadoopcmd:1,hand:[12,13],ha
 ndi:10,handl:[1,2,3],happen:[2,11,14],hard:[1,2],harmless:7,has_dag:1,hash:[1,12],hasn:[1,2],have:[1,2,3,4,5,6,7,8,9,11,12,14],haven:[1,2,14],hdf:[1,2,6],hdfs_conn_id:1,hdfs_default:1,hdfs_path:1,hdfscli:1,hdfssensor:1,head:[13,14],header:1,heartbeat:[1,2,4],heavier:1,hello:[2,8],help:[0,7,8,12],here:[1,2,3,4,6,8,10,11,13,14,15],herebi:7,herein:7,hex:1,hidden:15,hierarchi:[1,14],high:1,highchart:9,higher:1,highli:3,highlight:14,hipchat:1,hipchat_oper:1,hipchatapioper:1,hipchatapisendroomnotificationoper:1,histori:[],hit:[1,2],hive2mysqloper:1,hive2sambaoper:[1,6],hive:[1,2,3,6,8,12],hive_cli_conn_id:1,hive_cli_default:1,hive_cli_param:1,hive_conf:1,hive_conn_id:1,hive_t:1,hiveclihook:1,hivecmd:1,hiveconf:1,hiveconf_jinja_transl:1,hivemetastorehook:1,hiveoper:[1,2,3,12],hivepartitionsensor:[1,2],hivesensoroper:1,hiveserver2:1,hiveserver2_conn_id:1,hiveserver2_default:1,hiveserver2hook:1,hivetodruidtransf:1,hivetomysqltransf:1,hold:7,home:13,homogen:3,hope:11,horribl:14,host:[1,3,12],
 hostnam:[0,1,2,3],hour:[1,2,4,11],hourli:[1,4,11],hous:2,how:[],howev:[2,7,12],hql:1,html:[1,8,14],html_content:1,http:[1,2,7,8,12,14],http_conn_id:1,http_default:1,httphook:1,httpoper:2,httpsensor:1,human:1,icon:1,icon_url:1,idea:[],idempot:1,identif:7,identifi:[0,1,7,14,15],ignor:[0,1],ignore_depend:[0,1],ignore_depends_on_past:[0,1],ignore_first_depends_on_past:[0,1],iii:7,imag:1,immedi:[1,2],imper:3,imperson:12,implement:[1,2],impli:[1,7],improv:7,impyla:1,inabl:7,inact:[1,4],inc:7,incident:7,includ:[0,1,2,3,7,11,14],include_adhoc:[0,1],include_downstream:1,include_exampl:1,include_prior_d:1,include_queu:1,include_subdag:1,include_upstream:1,inclus:7,incorpor:7,incur:7,indemn:7,indemnifi:7,independ:[1,2],index:1,indic:[1,2,5,7],indirect:7,individu:[1,7,11,14],ineffici:1,infer:[1,2],inferred_op:2,infil:1,infin:5,info:1,inform:[1,2,3,7,11,12,14],infrastructur:[1,5],infring:7,ingest:1,inherit:[1,14],init:3,initctl:3,initdb:[0,3,13],initi:[0,3,12,13],initialis:1,inject:8,inlin:1,inp
 ut:1,input_format:1,insecur:12,insecurecli:1,insert:[1,2],insert_row:1,insid:[1,2],inspect:[11,13],instal:[],instanc:[],instanti:[],instead:[0,1,4],insteadnhttp:1,institut:7,instruct:2,insur:1,integr:[],intention:7,inter:2,interact:[1,2,3,8,9],interest:[1,14],intergr:8,intern:0,interpret:[1,2],interv:[1,4],intervalcheckoper:1,intoth:1,introduc:4,intuit:14,invari:[1,2],invok:1,ipython:2,irrevoc:7,is_paus:1,is_prematur:1,is_queueabl:1,is_runn:1,ish:1,isn:[],isoformat:1,issu:[1,2,5,7,10],item:2,iter:1,itself:[1,2,4,12,14],januari:[7,11],jar:1,java:1,jdbc:[1,6],jeremiah:10,jinja2:1,jinja:[],jinja_env:1,jlowin:10,job:[0,1,2,3,4,11,13,15],job_id:1,jobtrack:1,join:[2,10],json:[0,1,2],jupyt:2,just:[0,1,2,4,8,14],kadmin:12,kapil:3,keep:[],kei:[0,1,2,3,4,12,15],kerber:[6,12],kerbero:[],kernel:1,key1:1,key_fil:1,keytab:[0,12],keyword:1,kick:[3,11],kill:[1,2,4],kill_zombi:1,kind:7,know:[3,4,14],knowledg:8,known:[1,6],known_host:1,kruyk:10,kwarg:1,kwd:1,l447:1,lab:1,label:[1,9],lai:13,lambda:[1,
 2],land:[1,8],languag:[1,7,14],larg:1,larger:1,last:1,late:15,later:9,latest:[0,1,6,14],latest_d:1,latest_execution_d:1,latin_stat:1,launch:12,law:7,lawsuit:7,layer:4,layout:14,ldap3:12,ldap:[],ldap_auth:12,ldap_ca:12,lead:[1,2],lean:5,least:[1,2,7],leav:1,led:2,left:[2,12],legaci:4,legal:7,len:1,leo:10,less:1,lesson:10,let:[0,1,2,3,9,14,15],level:[1,2,5,11,12],leverag:[1,2,14],liabil:7,liabl:7,lib:[1,2,3],libffi:3,librari:[1,2,3,4,5,8,14],libz:2,licens:[],licensor:7,life:2,light:[1,2],lighter:1,lightweight:2,like:[1,2,3,4,6,8,12,14],limit:[],link:[],list:[],list_dag:[0,4,14],list_directori:1,list_kei:1,list_prefix:1,list_task:[0,4,14],listen:[2,4],lit:1,liter:1,litig:7,live:1,load:[0,1,2],load_fil:1,load_from_hdf:1,load_str:1,local:[0,1,2,3,4,12,14],local_executor:1,local_full_path_or_buff:1,local_infil:1,local_port:1,localexecutor:[0,1,2,3,4,11],localhost:2,locat:[0,1,2,3,12,14],lock:1,lock_for_upd:1,log:[],log_fil:0,logfil:0,loggin:1,loggingmixin:1,logic:[1,2,14],login:[1,2,3,12]
 ,logist:8,longer:[1,4],look:[0,1,2,4,5,8,10,11,14],loss:7,lot:11,low:1,lowin:10,machin:[1,2,3,12],made:[1,2,7],mai:[1,2,3,4,7,11,14],mail:[7,10],main:[1,2,4,8],main_dag:2,maintain:[1,5,10],make:[1,2,3,4,5,7,12,14,15],malfunct:7,malik:3,man:6,manag:[],mani:[0,1,2,4,8,14,15],manner:5,manual:[1,2,3,11],map:1,mapr:1,mapred_job_nam:1,mapred_queu:1,mapred_queue_prior:1,mark:[0,1,2,7,11],mark_success:[0,1,11],markdown:2,markup:1,master:[1,2,14],match:[0,1,2],matter:1,max:[1,10],max_active_run:[1,4],max_partit:1,max_retry_delai:1,maxim:10,maxime_beauchemin:0,maximum:1,mayb:2,mean:[1,2,3,4,7,12,14],meaning:1,meant:[1,4],mechan:[1,7,12],media:7,medium:7,meet:[1,7],mem_limit:1,member:12,memberof:12,memori:1,menu:[],menu_link:8,menulink:8,merchant:7,mere:7,merg:14,meso:[],mesos_executor:1,mesosexecutor:[1,3,11],messag:[1,2,5],message_format:1,met:[1,3,4,11],metadata:[],metastor:1,metastore_conn_id:1,metastore_default:1,metastore_mysql:1,metastorepartitionsensor:1,method:[1,2,4],metric:[1,8],met
 ric_spec:1,metrics_threshold:1,microsoft:[1,6],midnight:[4,11],might:[1,2,4,11,14],migrat:12,mime:1,mime_typ:1,min:4,mind:[],minimum:[1,6,11],minut:[2,4,11,14],mismatch:2,miss:[1,2],mistercrunch:10,mix:14,mkdir:2,mlsd:1,mode:[0,1,14],modif:7,modifi:[1,2,3,7,9],modul:[],modular:5,moment:4,monitor:[1,2,3,5,8,11,15],monospac:2,month:[4,11],monthli:[4,11],more:[1,2,3,5,7,8,11,12,14,15],morn:11,most:[1,2,12,14,15],mostli:[1,5,11],mount:[1,3],move:[1,2,4,5],msg:1,mssql:[1,6],mssql_conn_id:1,mssql_default:1,mssqlhook:1,mssqloper:1,mssqltohivetransf:1,much:[1,2,11],multi:[],multipart:1,multipart_byt:1,multipl:[1,2],multipli:1,multiprocess:1,must:[1,2,3,4,7,12,14],mutat:2,my_conn_str:3,my_dag1:2,my_dag2:2,my_dag:2,my_databas:1,my_dict_var:1,my_funct:2,my_param:14,my_tabl:1,my_useful_packag:2,my_var:1,myarg:1,mypackag:12,mys3conn:3,mysql:[1,2,3,6],mysql_conn_id:1,mysql_default:1,mysql_postoper:1,mysql_preoper:1,mysql_tabl:1,mysqldb:1,mysqlhook:1,mysqloper:[1,3],mysqltohivetransf:1,name:[1,2,3
 ,7,8,12],namedhivepartitionsensor:1,namespac:[1,4,8],nav:12,navig:12,ndfor:14,necessarili:[5,7],need:[1,2,3,4,5,6,8,11,12,13,14],neg:[1,11],neglig:7,neighbour:1,nest:[1,14],network:[1,12],network_mod:1,never:[0,1,4],new_user_email:12,new_user_nam:12,newli:5,newobject:1,next:[],next_retry_datetim:1,nhere:1,nice:2,night:2,nlst:1,no_confirm:0,no_host_key_check:1,node:[0,1,2],non:[1,3,7],none:[1,2,8,11],noram:12,norandkei:12,normal:[1,2,7],normalize_schedul:1,notat:1,note:[],notebook:2,noth:[1,7],notic:[2,7,14],notif:1,notifi:1,notwithstand:7,nov:10,now:[1,2,3,4,12,14],nuanc:2,num:1,num_run:0,num_shard:1,number:[0,1,2,3,5],oauth2:12,oauth_callback_rout:12,oauth_key_from_github_enterpris:12,oauth_secret_from_github_enterpris:12,obj:1,object:[0,1,2,3,4,7,8,11,14],objectclass:12,oblig:7,observ:1,obtain:7,occur:[1,2],occurr:1,octet:1,off:[1,3,11],offer:[7,8],offici:[1,10],offset:1,often:[1,2],on_failure_callback:1,on_kil:1,on_retry_callback:1,on_success_callback:1,onboard:4,onc:[1,2,3,4,11,
 14],one_fail:[1,2],one_success:[1,2],onli:[0,1,2,3,4,6,7,11,12,13],only_fail:[0,1],only_if_upd:1,only_run:[0,1],oozi:5,op1:2,op2:2,op3:2,op4:2,op_arg:1,op_kwarg:1,open:[1,2,4,10,12],oper:[],oppos:[1,2],optim:1,option:[],orchestr:[1,5],order:[1,2,3,11,12],org:[1,7,8,10,14],organ:[2,8],organiz:12,origin:7,orm:1,other:[1,2,3,5,7,9,11,14],otherwis:[1,7,14],our:14,out:[],outag:8,outgrow:13,outlier:15,output:[1,14],output_encod:1,output_format:1,outsid:11,outstand:7,over:[0,1,2,3,13,15],overlap:15,overrid:[0,1,14],overridden:[1,4],overview:15,overwhelm:2,overwrit:1,own:[],owner:[1,2,7,12,14],ownership:7,packag:[],package1:2,page:[2,4,7,10,15],pai:[1,4],pair:[1,15],panda:1,parallel:[1,2,3,13],param:[0,1,14],paramet:[0,1,2,3,4,9,14],parameter:[2,5,9],parent:[1,2],parent_dag:[1,2],parent_dag_nam:2,parent_modul:1,pars:[1,4,8,14],part:[1,2,7,9,12],parti:7,particular:7,partit:[1,2],partition_nam:1,partition_spec:1,partli:4,pass:[1,2,4,8,14],pass_valu:1,passwd:15,password:[],password_auth:12,pas
 swordus:12,past:[1,2,4,11,15],patent:7,path2:1,path:[1,2,3],patrick:10,patrickleotardif:10,pattern:[1,2],paus:[0,1],payload:1,pem:1,peopl:[1,8,14],pep:1,percent:7,perfect:2,perform:[0,1,2,3,4,5,7,15],perhap:[2,15],period:[1,2,4,11,14],permiss:[7,12],perpetu:7,persist:[1,11],perspect:2,pertain:7,phase:8,physic:1,pick:[2,3,9],pickl:[0,1,2,3],picklabl:1,pickle_id:1,pickleabl:1,pid:[0,13],pig:[1,2],pigcmd:1,piggi:8,pigoper:1,pin_100:1,pip:[2,6,13],pipelin:[],pitfal:10,pk1:1,pk2:1,place:[1,2,4,7,14],placehold:1,plain:[1,3],plan:4,platform:[1,2,5,6],pleas:[2,3,10,12,14],plug:8,plugin:[],pluginexecutor:8,pluginhook:8,pluginoper:8,plugins_manag:8,png:1,point:[1,2,3,4,13,14],poke:1,poke_interv:1,polymorph:1,pool:[],pool_ful:1,popen:1,popul:14,port:[0,1,3,12,13],posit:[0,1],possibl:[0,1,2,3,7,12,13,14,15],post:[1,10],post_execut:1,postgr:[1,2,3,6],postgres_conn_id:1,postgres_default:1,postgres_mast:2,postgreshook:1,postgresoper:1,potenti:2,power:[2,5,7,14],practic:[1,10],pre:4,pre_execut:1,pr
 eced:[1,3,14],prefer:[7,11,13],prefix:[1,2,3],preoper:1,prepar:[2,7],prepare_templ:1,prepped_request:1,present:1,preset:11,presto:1,presto_conn_id:1,presto_default:1,prestocheckoper:1,prestocmd:1,prestohook:1,prestointervalcheckoper:1,prestotomysqloper:2,prestovaluecheckoper:1,pretti:14,prevent:[1,2,12],previou:[0,1,2,4,14],previous:[2,4],primari:1,primit:1,princip:[0,2,12],print:[0,1,2,7,14],print_dat:14,print_head:1,prior:1,priorit:2,prioriti:[1,2],priority_weight:[1,2,14],privaci:7,probabl:[3,4,12],problem:2,problemat:2,proce:[1,4],process:[],process_fil:1,produc:1,product:[1,2,3,5,7,9,11,14],profil:[],program:[1,10],programmat:5,progress:[1,5,14],project:[],promin:7,prompt:0,propag:[1,2],propaget:1,proper:14,properli:[3,4],properti:[1,2,4,12],provid:[1,2,3,7,9,14,15],provide_context:[1,2],proxy_us:[1,12],proxyus:12,psycopg2:3,ptarget:1,publicli:7,publish:1,pull:[1,2],pull_funct:2,puppet:[3,10],pure:2,purpl:1,purpos:[7,14],push:[1,2,8],push_funct:2,pushing_task:2,put:[1,2,4,8],pv
 1:1,pv2:1,pwd:2,pydata:1,pyhiv:1,pypi:[6,13],python:[1,2,3,4,5,6,8,12,14],python_1:2,python_cal:[1,2],python_oper:1,pythonhost:8,pythonoper:[1,2],pythonpath:[3,12],qualif:12,qualifi:[1,12],qualiti:[1,2],qubol:[1,6],qubole_conn_id:1,qubole_default:1,quboleoper:1,queri:[],queu:[1,2,4],queue:[],quick:[],quickli:[9,13,14,15],quit:4,r39132:10,rabbitmq:[3,6],rais:[1,14],ran:0,randkei:12,random:1,randomli:2,rang:[0,1,4,11,14],rather:2,ratio:1,raw:[1,4],reach:[1,2,3,4],reactiv:4,read:[1,3,4,12,14],readabl:[1,7,12],readi:[1,5,13],readm:3,readthedoc:14,ready_for_retri:1,real:3,realli:[1,12,14],realm:12,reason:[4,7,14],reboot:2,rebuild:0,recap:[],receiv:[1,2,7,11],recent:[1,2],recip:10,recipi:[1,7],recogn:1,recommend:[1,3,4,7],record:[0,1,2,14],recreat:1,recurs:1,red:1,redhat:3,redi:3,redirect:0,redistribut:7,redund:14,referenc:[2,14],reflect:[2,5,8,14],refrain:2,refresh:1,refresh_from_db:1,regard:[1,7],regardless:[0,1],regarless:1,regex:[0,1],regex_kei:1,regexp:0,regist:[8,9,12],regular:[1,11
 ],reinit_frequ:12,rel:[1,11,14],relat:[1,2,3,4,6],relationship:2,relativedelta:1,relev:4,reli:[1,12],reliabl:1,remain:[7,12],rememb:[1,14],remot:[1,3,4,11],remote_base_log_fold:3,remote_full_path:1,remote_host:1,remote_log_conn_id:3,remote_port:1,remov:1,renam:1,render:[0,1,2,8,14],render_templ:1,render_template_from_field:1,renew:[0,12],repeat:2,repl:12,replac:[1,7,12],report:[0,1],repositori:[3,14],repres:[1,2,4,7,11],represent:[1,15],reproduc:7,reproduct:7,request:[0,1,2],requir:[1,2,3,4,5,6,7,12],resembl:2,reset:0,reset_dag_run:1,resetdb:0,resid:2,resolv:1,resourc:[],respect:[0,1,2,14],respons:[1,7],response_check:1,rest:1,restart:[2,3,4],restrict:12,restructuredtext:2,result:[1,2,7,9,14],resum:0,retain:7,retri:[1,2,14],retriev:[1,2],retrieve_fil:1,retry_delai:[1,14],retry_exponential_backoff:1,return_valu:1,reus:8,review:5,revis:7,riccomini:10,rich:[0,2,5,15],rid:1,right:[1,2,7,9],rightmost:2,risk:7,roadmap:[],role:3,roll:[],room:1,room_id:1,root:[1,2],root_dag:1,round:[1,4],ro
 ut:12,routin:[1,2],row:[1,2],royalti:7,rule:[],run:[],run_a:1,run_and_check:1,run_as_own:[1,12],run_cli:1,run_dur:0,run_id:[0,1,11],runme_0:13,runnabl:[1,2],rwc:12,s3_conn_id:1,s3_default:1,s3_kei:1,s3connect:1,s3distcp:1,s3filetransferoper:2,s3hook:[1,3],s3keysensor:[1,6],s3prefixsensor:6,s3tohivetransf:1,safe_mod:1,sai:[1,2,4,11],said:2,samaccountnam:12,samba:[1,6],samba_conn_id:1,samba_default:1,same:[1,2,3,7,9,12],sampl:[1,3],sample_s:1,sandbox:3,save:[1,3,4,9,14],scala:1,scalabl:5,scan:[1,2],scene:[6,11],schedul:[],schedule_interv:[1,2,4,11,14],scheduler_interv:4,scheduler_run:3,schema:[1,2,3],scratch:14,screen:14,screenshot:[],script:[],script_begin_tag:1,script_loc:1,search:[1,2],search_path:3,search_scop:12,second:[0,1,2,4,14],secret:[12,15],section:[1,2,3,4,7,12,13,14],secur:[],see:[1,2,3,4,7,11,12,13,14,15],seen:[3,15],segment:1,select:[1,2,9,11,12],self:[1,8],sell:7,semicolon:1,send:[0,1,2,10],send_room_notif:1,sender:1,sensor:[1,2,4],sensor_queu:2,sent:[1,2,3,7],separ:[1
 ,2,7,14],sequenti:[1,11,13],sequentialexecutor:[1,2,13],serial:[0,1],serv:[0,3,14],serve_log:[0,3],server:[0,1,3,8,12,13,14],servic:[0,1,2,3,6,7,11,12],session:[1,12],set:[],set_depend:[1,14],set_downstream:[1,2,14],set_the_password:12,set_upstream:[1,2,14],setgid:3,setuid:3,setup:[3,12],sever:2,shall:7,shape:11,share:[1,2,3,7,8,9],shell:[1,12],shellcmd:1,ship:[0,1],ship_dag:0,shop:7,shortcircuitoper:1,shortcut:[3,15],shortest:1,should:[1,2,3,4,7,8,11,13,14],shouldn:[1,2],show:[0,1,2,4,8,15],shown:1,sid:10,siddharth:10,side:[1,3],signal:[1,4],similar:[2,3,5],simpl:[1,2,4,8,9,14],simplehttpoper:1,simpler:14,simplest:12,simpli:[1,2,8,11,12,14],simplifi:1,simul:14,simultan:2,sinc:[1,3,13,14],sing:[],singl:[0,1,2,14],site:12,size:[1,2],skew:8,skip:[0,1,2],sla:[],sla_miss_callback:1,slack:[1,6],slack_oper:1,slackapioper:1,slackapipostoper:[1,6],slackoper:2,slave:3,sleep:14,slightli:5,slot:[1,2],slowli:5,slug:12,small:[1,2],smaller:1,smallish:1,snap:5,sock:1,soft_fail:1,softwar:[5,7,10],s
 ole:7,solut:5,some:[1,2,3,4,11,14,15],somehow:3,someth:[2,12,14],sometim:[1,2,14],somewhat:14,somewher:13,soon:[1,2,3,4,11],sort:2,sound:[2,14],sourc:[1,2,7,9,10,15],source_conn_id:1,source_project_dataset_t:1,space:5,span:15,spark:[1,2,5],sparkcmd:1,spawn:3,special:[1,2,4,7],specif:[0,1,2,3,4,7,11,14,15],specifi:[0,1,2,3,4,5,11,12,14],specific_valu:1,spent:15,split:1,split_column:1,sponsor:5,spring:10,sql:[1,2,3,6,9],sql_alchemy_conn:3,sql_alchemy_conn_cmd:3,sqlachemi:1,sqlalchemi:[1,2,3],sqlite:[1,13],sqlitehook:1,sqloper:2,sqlsensor:1,squeez:1,sscursor:1,ssh:[1,12],ssh_default:1,ssh_hook:1,sshexecuteoper:1,sshhook:1,sshpass:1,ssl:1,sslcert:1,sslmode:1,stabil:5,stabl:6,stack:[6,8],stage:[1,2],stai:11,stamp:11,stand:2,standalon:2,standard:[1,2,4],stanza:3,start:[],start_dat:[],stat:1,state:[0,1,2,4,7,11,14],state_d:1,statement:[1,2,7],static_babynam:1,static_babynames_partit:1,static_fold:8,static_path:1,static_url_path:8,statu:[0,1,2,3,4,5,7,11,13,14,15],stderr:0,stdout:[0,1,14],s
 tep:[1,3,14,15],steven:10,still:[],sting:1,stop:[1,3],stoppag:7,storag:[1,3],store:[0,1,2,3,8,12,13],store_fil:1,store_to_xcom_kei:1,storm:5,str:[1,8,11],straightforward:13,stream:[1,5],string:[0,1,2,3,14],string_data:1,structur:[1,5,14],style:1,sub:[0,1],sub_command:1,sub_commnad:1,sub_dag:[1,2],sub_part:1,subcommand:[1,3,11,14],subdag:[],subdag_oper:2,subdagoper:[1,2],subdir:0,subdirectori:2,subject:[1,7],sublicens:7,submiss:[7,11],submit:[1,7],subpackag:[6,12],subpartit:1,subprocess:[1,4,11],subscrib:[2,10],subsect:[0,11],subsequ:[0,7,11],subset:1,subtl:2,subtract:1,subtre:12,succe:[1,2],succeed:[0,1,2,4,15],success:[0,1,2,4,5,11,14],successfulli:[1,2,14],suit:5,sum:2,summit:10,sundai:11,super_us:12,supersed:7,superus:12,superuser_filt:12,suppli:[1,3,12],support:[0,1,3,6,7,12],sure:[1,2,3,4,12,14],surfac:14,surgeri:5,sync:[0,1,3,11],sync_tim:1,sync_to_db:1,synchron:[1,3],syntax:7,sysconfig:3,system:[1,2,3,7],syvineckruyk:10,tab:[1,12],tabl:[1,2,8],table_exist:1,table_nam:[1,2],ta
 blenam:1,tag:[1,4],take:[1,2,3,4,10,12,13,14,15],talk:[1,10,14],tardif:10,target:[1,3,11],target_field:1,target_partition_s:1,target_tim:1,task:[],task_id:[0,1,2,11,14],task_inst:[1,2],task_instance_key_str:1,task_or_task_list:1,task_param:0,task_regex:[0,1],task_stat:0,taskinst:[1,2],tbl:1,team:[1,12],tell:[0,1,4],templat:[],template_fold:8,template_searchpath:[1,14],templated_command:14,templates_dict:1,templates_ext:1,temporari:[1,2],tenanc:[],tenant:12,term:7,termin:[2,7],test:[],test_env:2,test_hql:1,test_mod:1,test_plugin:8,testabl:5,testview:8,text:[1,3,7,15],textfil:1,than:[1,2,5,14],thei:[0,1,2,3,5,6,8,9,11,14],them:[0,1,2,8,11,12,14],themselv:2,theoret:1,theori:[4,7],thereof:7,thi:[0,1,2,3,4,5,7,8,11,12,13,14,15],thin:1,thing:[1,2,4,14],think:[5,14],third:7,this_dag_will_be_discov:2,those:[1,2,3,7,12],though:[1,2,5],thousand:2,thread:1,three:[2,3],threshold:8,thrift:1,through:[1,2,6,7,11,13,14],throughout:1,thu:1,ticket:[0,12],tighten:12,time:[1,2,3,4,8,11,12,14,15],timede
 lta:[1,2,4,11,14],timedeltasensor:[1,4],timefram:1,timeout:[0,1,2],timesensor:1,timestamp:[1,2,11],tip:[2,10],titl:[2,7],tls_ca_cert:1,tls_client_cert:1,tls_client_kei:1,tls_hostnam:1,tls_ssl_version:1,tmp:[1,2],tmp_dir:1,tmp_file:1,to_nam:1,todai:1,togeth:[1,2],token:1,toler:[1,2],tomorrow:1,tomorrow_d:1,tomorrow_ds_nodash:1,too:[1,2,11],tool:[1,2,6,8],toolbox:8,top:[],topic:3,tornado:0,tort:7,touch:1,tour:[13,14],track:[2,7,14],tracker:1,trade:7,trademark:7,tradit:1,tradition:2,train:7,transact:1,transfer:[1,2,7],transform:7,translat:[1,4,7],transpar:15,travers:1,treat:[1,2],tree:[],tree_view:1,tri:[1,12],trick:10,trigger:[],trigger_dag:[0,11],trigger_dag_id:1,trigger_rul:[1,2],triggerdagrunoper:[1,10],triggerrul:1,troubl:6,troubleshoot:[5,15],truncat:1,truth:1,ts_dim:1,ts_nodash:1,tsv:1,tty:1,tunnel:[1,12],tupl:1,turn:[1,2,12],tutori:[],twice:1,two:[1,2,14],txt:1,type:[0,1,2,4,7,9,12],typic:[1,2],ubuntu:3,udf_config:1,uid:12,ultim:[1,14],unarchiv:1,unassign:2,unavail:3,unblock:11
 ,undefin:12,under:[1,2,3,7,8,10],undergo:5,underli:8,underscor:3,understand:[1,4,8,14,15],undesir:11,unexpect:2,unexpectedli:2,unicod:1,union:7,uniqu:[1,14],unit:[3,5,12],unittest:1,unix:1,unless:[1,4,7],unpack:[1,2],unpaus:0,unset:1,unsound:[1,2],unsuccess:1,unsupport:1,until:[1,2,3,5],up_for_retri:1,updat:[1,2,4,12],updateonli:1,upgrad:0,upgradedb:0,upload:1,upon:[2,3,13],upstream:[0,1,2,4,11],upstream_fail:[1,2],upstream_list:1,upstream_task_id:1,uri:[1,2,3,12],url:[1,2,3,8,9,12],usag:[0,1,14],use_beelin:[1,12],user:[0,1,2,3,5,6,12,14],user_defined_macro:1,user_filt:12,user_name_attr:12,user_program_argu:1,usernam:[1,3,12],usr:3,usual:[2,3],utf8:1,utf:1,util:[1,5,13],uuid:1,val:[0,1],valid:[],valu:[0,1,2,3,4,12,14,15],valuecheckoper:1,variabl:[],vast:1,verbal:7,verbos:[1,14],veri:[0,1,2,4,10,13,14],verifi:[2,4],version:[0,1,2,5,6,7],vertia:1,vertica:[1,6],vertica_conn_id:1,vertica_default:1,verticahook:1,verticaoper:1,verticaql:1,verticatohivetransf:1,very_high:1,very_low:1,via:[
 1,2,12],video:[1,10],view:[],virtual:1,virtualenv:2,visibl:[1,2],visual:[5,8,9,14,15],vocabulari:2,volum:1,wai:[1,2,3,4,6,8,11,12,14,15],wait:[0,1,2],wait_for_downstream:[1,4],wait_for_empty_queu:2,walk:[1,14],want:[1,2,3,4,8,11,12,14],warranti:7,wasn:[1,4],watch:[1,3,4],weapon:9,web:[],webhdfs_conn_id:1,webhdfs_default:1,webhdfshook:1,webhdfssensor:1,webserv:[0,3,4,12,13,14],week:11,weekli:11,weight:1,well:[1,2,3,4,11],were:1,what:[],whatev:[1,2,3,6],when:[1,2,3,4,5,8,9,11,12,14],whenev:1,where:[1,2,3,4,7,11,12,14,15],where_claus:1,wherev:[1,7],whether:[1,7,11],which:[0,1,2,3,4,7,11,12,13,14],whitelist:12,who:8,whole:[1,2,7],whom:7,whose:11,wiedmer:10,wiki:10,wildcard:1,wildcard_kei:1,wildcard_match:1,win:1,window:11,winter:10,wire:2,within:[0,1,2,3,4,7],without:[0,1,2,7,11,12],won:[1,4,6],word:[2,3,11,15],work:[0,1,2,3,4,5,7,8,9,11,12,13],worker:[0,1,2,3,4,5,6,8,12,14],worker_timeout:0,workerclass:0,workflow:[],workload:8,worldwid:7,worri:14,worth:1,would:[1,2,3,4,5,11,12,14],woul
 dn:[1,4],wrap:[1,14],wrapper:1,write:[3,5,7,8,9,14],write_disposit:1,write_empti:1,written:[1,7,14],wrong:14,www:[1,2,7],x01:1,xcom:[],xcom_al:1,xcom_pul:[1,2],xcom_push:[1,2],xcom_return_kei:1,xml:12,xst:12,xxx:1,yaml:2,year:[1,11],yearli:11,yellow:1,yesterdai:1,yesterday_d:1,yesterday_ds_nodash:1,yet:[1,2,5,12,15],you:[1,2,3,4,5,6,7,8,9,11,12,13,14,15],your:[],yourself:1,youtub:1,yum:6,yvinec:10,yyyi:[0,1,2],yyyymmdd:1,zero:1,zip:[1,2],zip_dag:2,zip_dag_cont:2,zoom:2},titles:["Command Line Interface","API Reference","Concepts","Configuration","FAQ","Apache Airflow (incubating) Documentation","Installation","License","Plugins","Data Profiling","Project","Scheduling & Triggers","Security","Quick Start","Tutorial","UI / Screenshots"],titleterms:{"default":[1,2,14],"function":2,"import":14,addit:2,adhoc:9,after:4,airflow:[4,5,6,8],all:4,anoth:4,apach:5,api:1,argument:[2,14],assign:2,authent:12,backend:3,backfil:14,base:4,baseoper:1,basesensoroper:1,beyond:5,bitshift:2,branch:2,bui
 ld:8,can:4,celeri:3,chart:[9,15],cluster:2,code:15,command:[0,4,14],committ:10,commun:[1,3],composit:2,concept:2,configur:3,connect:[2,3,4],content:5,context:[2,15],contribut:[1,3],core:2,creat:4,crypto:4,dag:[2,4,11,14,15],data:9,deal:4,definit:14,depend:14,document:[2,5],durat:15,dynam:4,enabl:12,encrypt:4,enterpris:12,exampl:[8,14],executor:1,extern:11,extra:6,failur:4,faq:4,file:14,form:9,gantt:15,get:[4,6],ghe:12,github:12,graph:15,histori:10,hook:[1,2],horizon:5,how:4,idea:2,incub:5,instal:[4,6],instanc:[2,15],instanti:14,integr:3,interfac:[0,8],isn:4,jinja:[2,14],keep:11,kerbero:12,ldap:12,licens:7,limit:12,line:[0,14],link:10,list:4,log:3,macro:1,manag:2,menu:15,meso:3,metadata:[4,14],mind:11,model:1,modul:14,multi:12,next:[13,14],note:2,oper:[1,2],option:3,out:3,own:12,packag:[2,6],password:[4,12],pipelin:14,plugin:8,polici:2,pool:2,principl:5,process:4,profil:9,project:10,queri:9,queue:2,quick:13,recap:14,refer:1,resourc:10,roadmap:10,roll:12,rule:2,run:[4,11,14],scale:3,s
 chedul:[4,11],scope:2,screenshot:[9,15],script:14,secur:12,set:[3,12,14],sla:2,start:13,start_dat:4,still:4,subdag:2,systemd:3,task:[2,4,14,15],templat:[2,14],tenanc:12,test:14,top:8,tree:15,trigger:[2,4,11],tutori:14,undead:2,upstart:3,valid:14,variabl:[1,2,15],view:15,web:12,what:[4,8,13,14],why:[4,8],workflow:2,xcom:2,your:12,zombi:2}})
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/security.html
----------------------------------------------------------------------
diff --git a/security.html b/security.html
index 8e65476..25ada3c 100644
--- a/security.html
+++ b/security.html
@@ -209,16 +209,20 @@ you will need to change <code class="docutils literal"><span class="pre">search_
 <span class="nv">auth_backend</span> <span class="o">=</span> airflow.contrib.auth.backends.ldap_auth
 
 <span class="o">[</span>ldap<span class="o">]</span>
+<span class="c1"># set a connection without encryption: uri = ldap://&lt;your.ldap.server&gt;:&lt;port&gt;</span>
 <span class="nv">uri</span> <span class="o">=</span> ldaps://&lt;your.ldap.server&gt;:&lt;port&gt;
 <span class="nv">user_filter</span> <span class="o">=</span> <span class="nv">objectClass</span><span class="o">=</span>*
-<span class="nv">user_name_attr</span> <span class="o">=</span> uid <span class="c1"># in case of Active Directory you would use sAMAccountName</span>
+<span class="c1"># in case of Active Directory you would use: user_name_attr = sAMAccountName</span>
+<span class="nv">user_name_attr</span> <span class="o">=</span> uid
 <span class="nv">superuser_filter</span> <span class="o">=</span> <span class="nv">memberOf</span><span class="o">=</span><span class="nv">CN</span><span class="o">=</span>airflow-super-users,OU<span class="o">=</span>Groups,OU<span class="o">=</span>RWC,OU<span class="o">=</span>US,OU<span class="o">=</span>NORAM,DC<span class="o">=</span>example,DC<span class="o">=</span>com
 <span class="nv">data_profiler_filter</span> <span class="o">=</span> <span class="nv">memberOf</span><span class="o">=</span><span class="nv">CN</span><span class="o">=</span>airflow-data-profilers,OU<span class="o">=</span>Groups,OU<span class="o">=</span>RWC,OU<span class="o">=</span>US,OU<span class="o">=</span>NORAM,DC<span class="o">=</span>example,DC<span class="o">=</span>com
 <span class="nv">bind_user</span> <span class="o">=</span> <span class="nv">cn</span><span class="o">=</span>Manager,dc<span class="o">=</span>example,dc<span class="o">=</span>com
 <span class="nv">bind_password</span> <span class="o">=</span> insecure
 <span class="nv">basedn</span> <span class="o">=</span> <span class="nv">dc</span><span class="o">=</span>example,dc<span class="o">=</span>com
 <span class="nv">cacert</span> <span class="o">=</span> /etc/ca/ldap_ca.crt
-<span class="nv">search_scope</span> <span class="o">=</span> LEVEL <span class="c1"># Set this to SUBTREE if using Active Directory, and not specifying an Organizational Unit</span>
+<span class="c1"># Set search_scope to one of them:  BASE, LEVEL , SUBTREE</span>
+<span class="c1"># Set search_scope to SUBTREE if using Active Directory, and not specifying an Organizational Unit</span>
+<span class="nv">search_scope</span> <span class="o">=</span> LEVEL
 </pre></div>
 </div>
 <p>The superuser_filter and data_profiler_filter are optional. If defined, these configurations allow you to specify LDAP groups that users must belong to in order to have superuser (admin) and data-profiler permissions. If undefined, all users will be superusers and data profilers.</p>

[4/7] incubator-airflow-site git commit: New version of the docs

[ma...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/presto_check_operator.html
----------------------------------------------------------------------
diff --git a/_modules/presto_check_operator.html b/_modules/presto_check_operator.html
index e24b271..639b71e 100644
--- a/_modules/presto_check_operator.html
+++ b/_modules/presto_check_operator.html
@@ -164,8 +164,9 @@
 <span class="sd">    values return ``False`` the check is failed and errors out.</span>
 
 <span class="sd">    Note that Python bool casting evals the following as ``False``:</span>
-<span class="sd">    * False</span>
-<span class="sd">    * 0</span>
+
+<span class="sd">    * ``False``</span>
+<span class="sd">    * ``0``</span>
 <span class="sd">    * Empty string (``&quot;&quot;``)</span>
 <span class="sd">    * Empty list (``[]``)</span>
 <span class="sd">    * Empty dictionary or set (``{}``)</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/sensors.html
----------------------------------------------------------------------
diff --git a/_modules/sensors.html b/_modules/sensors.html
index fab6313..6567e38 100644
--- a/_modules/sensors.html
+++ b/_modules/sensors.html
@@ -155,6 +155,8 @@
 <span class="kn">from</span> <span class="nn">future</span> <span class="kn">import</span> <span class="n">standard_library</span>
 <span class="n">standard_library</span><span class="o">.</span><span class="n">install_aliases</span><span class="p">()</span>
 <span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
+<span class="kn">from</span> <span class="nn">past.builtins</span> <span class="kn">import</span> <span class="nb">basestring</span>
+
 <span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">datetime</span>
 <span class="kn">import</span> <span class="nn">logging</span>
 <span class="kn">from</span> <span class="nn">urllib.parse</span> <span class="kn">import</span> <span class="n">urlparse</span>
@@ -229,6 +231,7 @@
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;sql&#39;</span><span class="p">,)</span>
     <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;.hql&#39;</span><span class="p">,</span> <span class="s1">&#39;.sql&#39;</span><span class="p">,)</span>
+    <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#7c7287&#39;</span>
 
     <span class="nd">@apply_defaults</span>
     <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">conn_id</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
@@ -272,6 +275,7 @@
 <span class="sd">    :type mysql_conn_id: str</span>
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;partition_name&#39;</span><span class="p">,</span> <span class="s1">&#39;table&#39;</span><span class="p">,</span> <span class="s1">&#39;schema&#39;</span><span class="p">)</span>
+    <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#8da7be&#39;</span>
 
     <span class="nd">@apply_defaults</span>
     <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
@@ -327,6 +331,7 @@
 <span class="sd">        or execution_date_fn can be passed to ExternalTaskSensor, but not both.</span>
 <span class="sd">    :type execution_date_fn: callable</span>
 <span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#19647e&#39;</span>
 
     <span class="nd">@apply_defaults</span>
     <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
@@ -376,16 +381,17 @@
         <span class="k">return</span> <span class="n">count</span></div>
 
 
-<span class="k">class</span> <span class="nc">NamedHivePartitionSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
+<div class="viewcode-block" id="NamedHivePartitionSensor"><a class="viewcode-back" href="../code.html#airflow.operators.NamedHivePartitionSensor">[docs]</a><span class="k">class</span> <span class="nc">NamedHivePartitionSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
     <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Waits for a set of partitions to show up in Hive.</span>
 
 <span class="sd">    :param partition_names: List of fully qualified names of the</span>
 <span class="sd">        partitions to wait for. A fully qualified name is of the</span>
-<span class="sd">        form schema.table/pk1=pv1/pk2=pv2, for example,</span>
+<span class="sd">        form ``schema.table/pk1=pv1/pk2=pv2``, for example,</span>
 <span class="sd">        default.users/ds=2016-01-01. This is passed as is to the metastore</span>
-<span class="sd">        Thrift client &quot;get_partitions_by_name&quot; method. Note that</span>
-<span class="sd">        you cannot use logical operators as in HivePartitionSensor.</span>
+<span class="sd">        Thrift client ``get_partitions_by_name`` method. Note that</span>
+<span class="sd">        you cannot use logical or comparison operators as in</span>
+<span class="sd">        HivePartitionSensor.</span>
 <span class="sd">    :type partition_names: list of strings</span>
 <span class="sd">    :param metastore_conn_id: reference to the metastore thrift service</span>
 <span class="sd">        connection id</span>
@@ -393,6 +399,7 @@
 <span class="sd">    &quot;&quot;&quot;</span>
 
     <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;partition_names&#39;</span><span class="p">,</span> <span class="p">)</span>
+    <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#8d99ae&#39;</span>
 
     <span class="nd">@apply_defaults</span>
     <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
@@ -408,9 +415,6 @@
         <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">partition_names</span><span class="p">,</span> <span class="nb">basestring</span><span class="p">):</span>
             <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s1">&#39;partition_names must be an array of strings&#39;</span><span class="p">)</span>
 
-        <span class="k">for</span> <span class="n">partition_name</span> <span class="ow">in</span> <span class="n">partition_names</span><span class="p">:</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">parse_partition_name</span><span class="p">(</span><span class="n">partition_name</span><span class="p">)</span>
-
         <span class="bp">self</span><span class="o">.</span><span class="n">metastore_conn_id</span> <span class="o">=</span> <span class="n">metastore_conn_id</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">partition_names</span> <span class="o">=</span> <span class="n">partition_names</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">next_poke_idx</span> <span class="o">=</span> <span class="mi">0</span>
@@ -445,14 +449,14 @@
             <span class="k">else</span><span class="p">:</span>
                 <span class="k">return</span> <span class="bp">False</span>
 
-        <span class="k">return</span> <span class="bp">True</span>
+        <span class="k">return</span> <span class="bp">True</span></div>
 
 
 <div class="viewcode-block" id="HivePartitionSensor"><a class="viewcode-back" href="../code.html#airflow.operators.HivePartitionSensor">[docs]</a><span class="k">class</span> <span class="nc">HivePartitionSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
     <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Waits for a partition to show up in Hive.</span>
 
-<span class="sd">    Note: Because @partition supports general logical operators, it</span>
+<span class="sd">    Note: Because ``partition`` supports general logical operators, it</span>
 <span class="sd">    can be inefficient. Consider using NamedHivePartitionSensor instead if</span>
 <span class="sd">    you don&#39;t need the full flexibility of HivePartitionSensor.</span>
 
@@ -460,15 +464,16 @@
 <span class="sd">        notation (my_database.my_table)</span>
 <span class="sd">    :type table: string</span>
 <span class="sd">    :param partition: The partition clause to wait for. This is passed as</span>
-<span class="sd">        is to the metastore Thrift client &quot;get_partitions_by_filter&quot; method,</span>
-<span class="sd">        and apparently supports SQL like notation as in `ds=&#39;2015-01-01&#39;</span>
-<span class="sd">        AND type=&#39;value&#39;` and &gt; &lt; sings as in &quot;ds&gt;=2015-01-01&quot;</span>
+<span class="sd">        is to the metastore Thrift client ``get_partitions_by_filter`` method,</span>
+<span class="sd">        and apparently supports SQL like notation as in ``ds=&#39;2015-01-01&#39;</span>
+<span class="sd">        AND type=&#39;value&#39;`` and comparison operators as in ``&quot;ds&gt;=2015-01-01&quot;``</span>
 <span class="sd">    :type partition: string</span>
 <span class="sd">    :param metastore_conn_id: reference to the metastore thrift service</span>
 <span class="sd">        connection id</span>
 <span class="sd">    :type metastore_conn_id: str</span>
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;schema&#39;</span><span class="p">,</span> <span class="s1">&#39;table&#39;</span><span class="p">,</span> <span class="s1">&#39;partition&#39;</span><span class="p">,)</span>
+    <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#2b2d42&#39;</span>
 
     <span class="nd">@apply_defaults</span>
     <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
@@ -505,6 +510,7 @@
 <span class="sd">    Waits for a file or folder to land in HDFS</span>
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;filepath&#39;</span><span class="p">,)</span>
+    <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#4d9de0&#39;</span>
 
     <span class="nd">@apply_defaults</span>
     <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/ssh_hook.html
----------------------------------------------------------------------
diff --git a/_modules/ssh_hook.html b/_modules/ssh_hook.html
index 7f21905..e094b62 100644
--- a/_modules/ssh_hook.html
+++ b/_modules/ssh_hook.html
@@ -212,7 +212,7 @@
         <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">sshpass</span><span class="p">:</span>
             <span class="n">connection_cmd</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;sshpass&quot;</span><span class="p">,</span> <span class="s2">&quot;-e&quot;</span><span class="p">]</span> <span class="o">+</span> <span class="n">connection_cmd</span>
         <span class="k">else</span><span class="p">:</span>
-            <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">&quot;-o&quot;</span><span class="p">,</span> <span class="s2">&quot;BatchMode=yes&quot;</span><span class="p">]</span> <span class="c1"># no password prompts</span>
+            <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">&quot;-o&quot;</span><span class="p">,</span> <span class="s2">&quot;BatchMode=yes&quot;</span><span class="p">]</span>  <span class="c1"># no password prompts</span>
 
         <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">:</span>
             <span class="n">connection_cmd</span> <span class="o">+=</span> <span class="p">[</span><span class="s2">&quot;-p&quot;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">)]</span>
@@ -281,8 +281,7 @@
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="n">tunnel_host</span> <span class="o">=</span> <span class="s2">&quot;{0}:{1}:{2}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">local_port</span><span class="p">,</span> <span class="n">remote_host</span><span class="p">,</span> <span class="n">remote_port</span><span class="p">)</span>
         <span class="n">proc</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">Popen</span><span class="p">([</span><span class="s2">&quot;-L&quot;</span><span class="p">,</span> <span class="n">tunnel_host</span><span class="p">,</span> <span class="s2">&quot;echo -n ready &amp;&amp; cat&quot;</span><span class="p">],</span>
-                           <span class="n">stdin</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">PIPE</span><span class="p">,</span> <span class="n">stdout</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">PIPE</span><span class="p">,</span>
-                           <span class="p">)</span>
+                          <span class="n">stdin</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">PIPE</span><span class="p">,</span> <span class="n">stdout</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">PIPE</span><span class="p">)</span>
 
         <span class="n">ready</span> <span class="o">=</span> <span class="n">proc</span><span class="o">.</span><span class="n">stdout</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="mi">5</span><span class="p">)</span>
         <span class="k">assert</span> <span class="n">ready</span> <span class="o">==</span> <span class="n">b</span><span class="s2">&quot;ready&quot;</span><span class="p">,</span> <span class="s2">&quot;Did not get &#39;ready&#39; from remote&quot;</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/vertica_hook.html
----------------------------------------------------------------------
diff --git a/_modules/vertica_hook.html b/_modules/vertica_hook.html
index 7ec4ec0..dab06bd 100644
--- a/_modules/vertica_hook.html
+++ b/_modules/vertica_hook.html
@@ -156,6 +156,7 @@
 
 <span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="kn">import</span> <span class="n">DbApiHook</span>
 
+
 <div class="viewcode-block" id="VerticaHook"><a class="viewcode-back" href="../code.html#airflow.contrib.hooks.VerticaHook">[docs]</a><span class="k">class</span> <span class="nc">VerticaHook</span><span class="p">(</span><span class="n">DbApiHook</span><span class="p">):</span>
     <span class="sd">&#39;&#39;&#39;</span>
 <span class="sd">    Interact with Vertica.</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/vertica_to_hive.html
----------------------------------------------------------------------
diff --git a/_modules/vertica_to_hive.html b/_modules/vertica_to_hive.html
index a700374..a8585d1 100644
--- a/_modules/vertica_to_hive.html
+++ b/_modules/vertica_to_hive.html
@@ -162,6 +162,7 @@
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 
+
 <div class="viewcode-block" id="VerticaToHiveTransfer"><a class="viewcode-back" href="../code.html#airflow.contrib.operators.VerticaToHiveTransfer">[docs]</a><span class="k">class</span> <span class="nc">VerticaToHiveTransfer</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
     <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Moves data from Vertia to Hive. The operator runs</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/webhdfs_hook.html
----------------------------------------------------------------------
diff --git a/_modules/webhdfs_hook.html b/_modules/webhdfs_hook.html
index 3a44028..70c641c 100644
--- a/_modules/webhdfs_hook.html
+++ b/_modules/webhdfs_hook.html
@@ -159,11 +159,11 @@
 
 <span class="n">_kerberos_security_mode</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;core&quot;</span><span class="p">,</span> <span class="s2">&quot;security&quot;</span><span class="p">)</span> <span class="o">==</span> <span class="s2">&quot;kerberos&quot;</span>
 <span class="k">if</span> <span class="n">_kerberos_security_mode</span><span class="p">:</span>
-  <span class="k">try</span><span class="p">:</span>
-    <span class="kn">from</span> <span class="nn">hdfs.ext.kerberos</span> <span class="kn">import</span> <span class="n">KerberosClient</span>
-  <span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span>
-    <span class="n">logging</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;Could not load the Kerberos extension for the WebHDFSHook.&quot;</span><span class="p">)</span>
-    <span class="k">raise</span>
+    <span class="k">try</span><span class="p">:</span>
+        <span class="kn">from</span> <span class="nn">hdfs.ext.kerberos</span> <span class="kn">import</span> <span class="n">KerberosClient</span>
+    <span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span>
+        <span class="n">logging</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;Could not load the Kerberos extension for the WebHDFSHook.&quot;</span><span class="p">)</span>
+        <span class="k">raise</span>
 <span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span>
 
 
@@ -189,10 +189,10 @@
                 <span class="n">logging</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">&#39;Trying namenode {}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">host</span><span class="p">))</span>
                 <span class="n">connection_str</span> <span class="o">=</span> <span class="s1">&#39;http://{nn.host}:{nn.port}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">nn</span><span class="o">=</span><span class="n">nn</span><span class="p">)</span>
                 <span class="k">if</span> <span class="n">_kerberos_security_mode</span><span class="p">:</span>
-                  <span class="n">client</span> <span class="o">=</span> <span class="n">KerberosClient</span><span class="p">(</span><span class="n">connection_str</span><span class="p">)</span>
+                    <span class="n">client</span> <span class="o">=</span> <span class="n">KerberosClient</span><span class="p">(</span><span class="n">connection_str</span><span class="p">)</span>
                 <span class="k">else</span><span class="p">:</span>
-                  <span class="n">proxy_user</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">proxy_user</span> <span class="ow">or</span> <span class="n">nn</span><span class="o">.</span><span class="n">login</span>
-                  <span class="n">client</span> <span class="o">=</span> <span class="n">InsecureClient</span><span class="p">(</span><span class="n">connection_str</span><span class="p">,</span> <span class="n">user</span><span class="o">=</span><span class="n">proxy_user</span><span class="p">)</span>
+                    <span class="n">proxy_user</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">proxy_user</span> <span class="ow">or</span> <span class="n">nn</span><span class="o">.</span><span class="n">login</span>
+                    <span class="n">client</span> <span class="o">=</span> <span class="n">InsecureClient</span><span class="p">(</span><span class="n">connection_str</span><span class="p">,</span> <span class="n">user</span><span class="o">=</span><span class="n">proxy_user</span><span class="p">)</span>
                 <span class="n">client</span><span class="o">.</span><span class="n">status</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">)</span>
                 <span class="n">logging</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">&#39;Using namenode {} for hook&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">host</span><span class="p">))</span>
                 <span class="k">return</span> <span class="n">client</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_sources/code.txt
----------------------------------------------------------------------
diff --git a/_sources/code.txt b/_sources/code.txt
index ba1cddb..8548120 100644
--- a/_sources/code.txt
+++ b/_sources/code.txt
@@ -66,6 +66,7 @@ Operator API
         MsSqlToHiveTransfer,
         MySqlOperator,
         MySqlToHiveTransfer,
+        NamedHivePartitionSensor,
         PostgresOperator,
         PrestoCheckOperator,
         PrestoIntervalCheckOperator,

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_sources/faq.txt
----------------------------------------------------------------------
diff --git a/_sources/faq.txt b/_sources/faq.txt
index e61c1bf..68b3189 100644
--- a/_sources/faq.txt
+++ b/_sources/faq.txt
@@ -1,7 +1,8 @@
 FAQ
 ========
 
-**Why isn't my task getting scheduled?**
+Why isn't my task getting scheduled?
+------------------------------------
 
 There are very many reasons why your task might not be getting scheduled.
 Here are some of the common causes:
@@ -51,17 +52,20 @@ You may also want to read the Scheduler section of the docs and make
 sure you fully understand how it proceeds.
 
 
-**How do I trigger tasks based on another task's failure?**
+How do I trigger tasks based on another task's failure?
+-------------------------------------------------------
 
 Check out the ``Trigger Rule`` section in the Concepts section of the
 documentation
 
-**Why are connection passwords still not encrypted in the metadata db after I installed airflow[crypto]**?
+Why are connection passwords still not encrypted in the metadata db after I installed airflow[crypto]?
+------------------------------------------------------------------------------------------------------
 
 - Verify that the ``fernet_key`` defined in ``$AIRFLOW_HOME/airflow.cfg`` is a valid Fernet key. It must be a base64-encoded 32-byte key. You need to restart the webserver after you update the key
 - For existing connections (the ones that you had defined before installing ``airflow[crypto]`` and creating a Fernet key), you need to open each connection in the connection admin UI, re-type the password, and save it
 
-**What's the deal with ``start_date``?**
+What's the deal with ``start_date``?
+------------------------------------
 
 ``start_date`` is partly legacy from the pre-DagRun era, but it is still
 relevant in many ways. When creating a new DAG, you probably want to set
@@ -82,7 +86,7 @@ an hour after now as ``now()`` moves along.
 Previously we also recommended using rounded ``start_date`` in relation to your
 ``schedule_interval``. This meant an ``@hourly`` would be at ``00:00``
 minutes:seconds, a ``@daily`` job at midnight, a ``@monthly`` job on the
-first of the month. This is no longer required. Airflow will not auto align
+first of the month. This is no longer required. Airflow will now auto align
 the ``start_date`` and the ``schedule_interval``, by using the ``start_date``
 as the moment to start looking.
 
@@ -103,3 +107,36 @@ Also important to note is that the tasks ``start_date``, in the context of a
 backfill CLI command, get overridden by the backfill's command ``start_date``.
 This allows for a backfill on tasks that have ``depends_on_past=True`` to
 actually start, if it wasn't the case, the backfill just wouldn't start.
+
+How can I create DAGs dynamically?
+----------------------------------
+
+Airflow looks in you ``DAGS_FOLDER`` for modules that contain ``DAG`` objects
+in their global namespace, and adds the objects it finds in the
+``DagBag``. Knowing this all we need is a way to dynamically assign
+variable in the global namespace, which is easily done in python using the
+``globals()`` function for the standard library which behaves like a
+simple dictionary.
+
+.. code:: python
+
+    for i in range(10):
+        dag_id = 'foo_{}'.format(i)
+        globals()[dag_id] = DAG(dag_id)
+        # or better, call a function that returns a DAG object!
+
+What are all the ``airflow run`` commands in my process list?
+---------------------------------------------------------------
+
+There are many layers of ``airflow run`` commands, meaning it can call itself.
+
+- Basic ``airflow run``: fires up an executor, and tell it to run an
+  ``airflow run --local`` command. if using Celery, this means it puts a
+  command in the queue for it to run remote, on the worker. If using
+  LocalExecutor, that translates into running it in a subprocess pool.
+- Local ``airflow run --local``: starts an ``airflow run --raw``
+  command (described bellow) as a subprocess and is in charge of
+  emitting heartbeats, listening for external kill signals
+  and ensures some cleanup takes place if the subprocess fails
+- Raw ``airflow run --raw`` runs the actual operator's execute method and
+  performs the actual work

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_sources/security.txt
----------------------------------------------------------------------
diff --git a/_sources/security.txt b/_sources/security.txt
index 1f33548..50d9167 100644
--- a/_sources/security.txt
+++ b/_sources/security.txt
@@ -67,16 +67,20 @@ Valid search_scope options can be found in the `ldap3 Documentation <http://ldap
     auth_backend = airflow.contrib.auth.backends.ldap_auth
 
     [ldap]
+    # set a connection without encryption: uri = ldap://<your.ldap.server>:<port>
     uri = ldaps://<your.ldap.server>:<port>
     user_filter = objectClass=*
-    user_name_attr = uid # in case of Active Directory you would use sAMAccountName
+    # in case of Active Directory you would use: user_name_attr = sAMAccountName
+    user_name_attr = uid
     superuser_filter = memberOf=CN=airflow-super-users,OU=Groups,OU=RWC,OU=US,OU=NORAM,DC=example,DC=com
     data_profiler_filter = memberOf=CN=airflow-data-profilers,OU=Groups,OU=RWC,OU=US,OU=NORAM,DC=example,DC=com
     bind_user = cn=Manager,dc=example,dc=com
     bind_password = insecure
     basedn = dc=example,dc=com
     cacert = /etc/ca/ldap_ca.crt
-    search_scope = LEVEL # Set this to SUBTREE if using Active Directory, and not specifying an Organizational Unit
+    # Set search_scope to one of them:  BASE, LEVEL , SUBTREE
+    # Set search_scope to SUBTREE if using Active Directory, and not specifying an Organizational Unit
+    search_scope = LEVEL
 
 The superuser_filter and data_profiler_filter are optional. If defined, these configurations allow you to specify LDAP groups that users must belong to in order to have superuser (admin) and data-profiler permissions. If undefined, all users will be superusers and data profilers.
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_sources/tutorial.txt
----------------------------------------------------------------------
diff --git a/_sources/tutorial.txt b/_sources/tutorial.txt
index a93479c..97bbe11 100644
--- a/_sources/tutorial.txt
+++ b/_sources/tutorial.txt
@@ -39,7 +39,7 @@ complicated, a line by line explanation follows below.
 
     dag = DAG('tutorial', default_args=default_args)
 
-    # t1, t2 and t3 are examples of tasks created by instatiating operators
+    # t1, t2 and t3 are examples of tasks created by instantiating operators
     t1 = BashOperator(
         task_id='print_date',
         bash_command='date',
@@ -292,7 +292,7 @@ something like this:
     dag = DAG(
         'tutorial', default_args=default_args, schedule_interval=timedelta(1))
 
-    # t1, t2 and t3 are examples of tasks created by instatiating operators
+    # t1, t2 and t3 are examples of tasks created by instantiating operators
     t1 = BashOperator(
         task_id='print_date',
         bash_command='date',

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_static/default.css
----------------------------------------------------------------------
diff --git a/_static/default.css b/_static/default.css
new file mode 100644
index 0000000..4aef108
--- /dev/null
+++ b/_static/default.css
@@ -0,0 +1,3 @@
+.admonition {
+    background-color: gray;
+}

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/cli.html
----------------------------------------------------------------------
diff --git a/cli.html b/cli.html
index e8785c4..71ebad4 100644
--- a/cli.html
+++ b/cli.html
@@ -210,6 +210,7 @@ development and testing.</p>
 <dt><strong>variables</strong></dt>
 <dd><p class="first">List all variables</p>
 <div class="highlight-default"><div class="highlight"><pre><span></span><span class="n">usage</span><span class="p">:</span> <span class="n">airflow</span> <span class="n">variables</span> <span class="p">[</span><span class="o">-</span><span class="n">h</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">s</span> <span class="n">KEY</span> <span class="n">VAL</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">g</span> <span class="n">KEY</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">j</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">d</span> <span class="n">VAL</span><span class="p">]</span>
+                         <span class="p">[</span><span class="o">-</span><span class="n">i</span> <span class="n">FILEPATH</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">e</span> <span class="n">FILEPATH</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">x</span> <span class="n">KEY</span><span class="p">]</span>
 </pre></div>
 </div>
 <dl class="last docutils">
@@ -231,6 +232,15 @@ development and testing.</p>
 <tr><td class="option-group">
 <kbd><span class="option">-d</span>, <span class="option">--default</span></kbd></td>
 <td>Default value returned if variable does not exist</td></tr>
+<tr><td class="option-group">
+<kbd><span class="option">-i</span>, <span class="option">--import</span></kbd></td>
+<td>Import variables from JSON file</td></tr>
+<tr><td class="option-group">
+<kbd><span class="option">-e</span>, <span class="option">--export</span></kbd></td>
+<td>Export variables to JSON file</td></tr>
+<tr><td class="option-group">
+<kbd><span class="option">-x</span>, <span class="option">--delete</span></kbd></td>
+<td>Delete a variable</td></tr>
 </tbody>
 </table>
 </dd>
@@ -765,8 +775,9 @@ development and testing.</p>
 </dd>
 <dt><strong>flower</strong></dt>
 <dd><p class="first">Start a Celery Flower</p>
-<div class="highlight-default"><div class="highlight"><pre><span></span><span class="n">usage</span><span class="p">:</span> <span class="n">airflow</span> <span class="n">flower</span> <span class="p">[</span><span class="o">-</span><span class="n">h</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">p</span> <span class="n">PORT</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">a</span> <span class="n">BROKER_API</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">pid</span> <span class="p">[</span><span class="n">PID</span><span class="p">]]</span> <span class="p">[</span><span class="o">-</span><span class="n">D</span><span class="p">]</span>
-                      <span class="p">[</span><span class="o">--</span><span class="n">stdout</span> <span class="n">STDOUT</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">stderr</span> <span class="n">STDERR</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">l</span> <span class="n">LOG_FILE</span><span class="p">]</span>
+<div class="highlight-default"><div class="highlight"><pre><span></span><span class="n">usage</span><span class="p">:</span> <span class="n">airflow</span> <span class="n">flower</span> <span class="p">[</span><span class="o">-</span><span class="n">h</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">hn</span> <span class="n">HOSTNAME</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">p</span> <span class="n">PORT</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">fc</span> <span class="n">FLOWER_CONF</span><span class="p">]</span>
+                      <span class="p">[</span><span class="o">-</span><span class="n">a</span> <span class="n">BROKER_API</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">pid</span> <span class="p">[</span><span class="n">PID</span><span class="p">]]</span> <span class="p">[</span><span class="o">-</span><span class="n">D</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">stdout</span> <span class="n">STDOUT</span><span class="p">]</span>
+                      <span class="p">[</span><span class="o">--</span><span class="n">stderr</span> <span class="n">STDERR</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">l</span> <span class="n">LOG_FILE</span><span class="p">]</span>
 </pre></div>
 </div>
 <dl class="last docutils">
@@ -776,10 +787,18 @@ development and testing.</p>
 <col class="description" />
 <tbody valign="top">
 <tr><td class="option-group" colspan="2">
+<kbd><span class="option">-hn=0.0.0.0</span>, <span class="option">--hostname=0.0.0.0</span></kbd></td>
+</tr>
+<tr><td>&nbsp;</td><td>Set the hostname on which to run the server</td></tr>
+<tr><td class="option-group" colspan="2">
 <kbd><span class="option">-p=5555</span>, <span class="option">--port=5555</span></kbd></td>
 </tr>
 <tr><td>&nbsp;</td><td>The port on which to run the server</td></tr>
 <tr><td class="option-group" colspan="2">
+<kbd><span class="option">-fc</span>, <span class="option">--flower_conf</span></kbd></td>
+</tr>
+<tr><td>&nbsp;</td><td>Configuration file for flower</td></tr>
+<tr><td class="option-group" colspan="2">
 <kbd><span class="option">-a</span>, <span class="option">--broker_api</span></kbd></td>
 </tr>
 <tr><td>&nbsp;</td><td>Broker api</td></tr>
@@ -806,9 +825,9 @@ development and testing.</p>
 </dd>
 <dt><strong>scheduler</strong></dt>
 <dd><p class="first">Start a scheduler instance</p>
-<div class="highlight-default"><div class="highlight"><pre><span></span><span class="n">usage</span><span class="p">:</span> <span class="n">airflow</span> <span class="n">scheduler</span> <span class="p">[</span><span class="o">-</span><span class="n">h</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">d</span> <span class="n">DAG_ID</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">sd</span> <span class="n">SUBDIR</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">n</span> <span class="n">NUM_RUNS</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">p</span><span class="p">]</span>
-                         <span class="p">[</span><span class="o">--</span><span class="n">pid</span> <span class="p">[</span><span class="n">PID</span><span class="p">]]</span> <span class="p">[</span><span class="o">-</span><span class="n">D</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">stdout</span> <span class="n">STDOUT</span><span class="p">]</span>
-                         <span class="p">[</span><span class="o">--</span><span class="n">stderr</span> <span class="n">STDERR</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">l</span> <span class="n">LOG_FILE</span><span class="p">]</span>
+<div class="highlight-default"><div class="highlight"><pre><span></span><span class="n">usage</span><span class="p">:</span> <span class="n">airflow</span> <span class="n">scheduler</span> <span class="p">[</span><span class="o">-</span><span class="n">h</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">d</span> <span class="n">DAG_ID</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">sd</span> <span class="n">SUBDIR</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">r</span> <span class="n">RUN_DURATION</span><span class="p">]</span>
+                         <span class="p">[</span><span class="o">-</span><span class="n">n</span> <span class="n">NUM_RUNS</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">p</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">pid</span> <span class="p">[</span><span class="n">PID</span><span class="p">]]</span> <span class="p">[</span><span class="o">-</span><span class="n">D</span><span class="p">]</span>
+                         <span class="p">[</span><span class="o">--</span><span class="n">stdout</span> <span class="n">STDOUT</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">stderr</span> <span class="n">STDERR</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">l</span> <span class="n">LOG_FILE</span><span class="p">]</span>
 </pre></div>
 </div>
 <dl class="last docutils">
@@ -824,6 +843,10 @@ development and testing.</p>
 <kbd><span class="option">-sd=/Users/maxime_beauchemin/dags</span>, <span class="option">--subdir=/Users/maxime_beauchemin/dags</span></kbd></td>
 </tr>
 <tr><td>&nbsp;</td><td>File location or directory from which to look for the dag</td></tr>
+<tr><td class="option-group" colspan="2">
+<kbd><span class="option">-r</span>, <span class="option">--run-duration</span></kbd></td>
+</tr>
+<tr><td>&nbsp;</td><td>Set number of seconds to execute before exiting</td></tr>
 <tr><td class="option-group">
 <kbd><span class="option">-n</span>, <span class="option">--num_runs</span></kbd></td>
 <td>Set the number of runs to execute before exiting</td></tr>

[7/7] incubator-airflow-site git commit: New version of the docs

[ma...@apache.org]

New version of the docs


Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/commit/4af0850c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/tree/4af0850c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/diff/4af0850c

Branch: refs/heads/asf-site
Commit: 4af0850c3b9f8d0874a58a51ddf5cf83c4ad74e5
Parents: 457b698
Author: Maxime Beauchemin <ma...@apache.org>
Authored: Thu Aug 11 15:58:56 2016 -0700
Committer: Maxime Beauchemin <ma...@apache.org>
Committed: Thu Aug 11 15:59:05 2016 -0700

----------------------------------------------------------------------
 _modules/S3_hook.html                           |   1 +
 .../contrib/operators/bigquery_operator.html    |   1 +
 .../contrib/operators/bigquery_to_gcs.html      |   1 +
 .../operators/gcs_download_operator.html        |   1 +
 _modules/airflow/executors/local_executor.html  |   4 +-
 _modules/airflow/macros.html                    |   1 -
 _modules/airflow/models.html                    | 408 ++++++++++++++++---
 _modules/airflow/operators/sensors.html         |  26 +-
 _modules/bash_operator.html                     |   8 +-
 _modules/bigquery_hook.html                     |  56 +--
 _modules/cloudant_hook.html                     |   4 +-
 _modules/dagrun_operator.html                   |   1 +
 _modules/dbapi_hook.html                        |  15 +-
 _modules/druid_hook.html                        |  14 +-
 _modules/email_operator.html                    |  10 +-
 _modules/ftp_hook.html                          |  10 +
 _modules/hive_hooks.html                        | 183 ++++++---
 _modules/hive_operator.html                     |  22 +-
 _modules/hive_to_mysql.html                     |   1 +
 _modules/mysql_hook.html                        |   1 +
 _modules/presto_check_operator.html             |   5 +-
 _modules/sensors.html                           |  30 +-
 _modules/ssh_hook.html                          |   5 +-
 _modules/vertica_hook.html                      |   1 +
 _modules/vertica_to_hive.html                   |   1 +
 _modules/webhdfs_hook.html                      |  16 +-
 _sources/code.txt                               |   1 +
 _sources/faq.txt                                |  47 ++-
 _sources/security.txt                           |   8 +-
 _sources/tutorial.txt                           |   4 +-
 _static/default.css                             |   3 +
 cli.html                                        |  33 +-
 code.html                                       | 309 +++++++++++---
 faq.html                                        |  59 ++-
 genindex.html                                   |  48 ++-
 index.html                                      |  10 +-
 objects.inv                                     | Bin 2246 -> 2326 bytes
 searchindex.js                                  |   2 +-
 security.html                                   |   8 +-
 tutorial.html                                   |   4 +-
 40 files changed, 1079 insertions(+), 283 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/S3_hook.html
----------------------------------------------------------------------
diff --git a/_modules/S3_hook.html b/_modules/S3_hook.html
index cca8dea..40a14c3 100644
--- a/_modules/S3_hook.html
+++ b/_modules/S3_hook.html
@@ -151,6 +151,7 @@
 <span class="c1"># See the License for the specific language governing permissions and</span>
 <span class="c1"># limitations under the License.</span>
 
+<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">division</span>
 <span class="kn">from</span> <span class="nn">future</span> <span class="kn">import</span> <span class="n">standard_library</span>
 <span class="n">standard_library</span><span class="o">.</span><span class="n">install_aliases</span><span class="p">()</span>
 <span class="kn">import</span> <span class="nn">logging</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/airflow/contrib/operators/bigquery_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/bigquery_operator.html b/_modules/airflow/contrib/operators/bigquery_operator.html
index 6038f2c..c7fcfd9 100644
--- a/_modules/airflow/contrib/operators/bigquery_operator.html
+++ b/_modules/airflow/contrib/operators/bigquery_operator.html
@@ -157,6 +157,7 @@
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 
+
 <div class="viewcode-block" id="BigQueryOperator"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.bigquery_operator.BigQueryOperator">[docs]</a><span class="k">class</span> <span class="nc">BigQueryOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
     <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Executes BigQuery SQL queries in a specific BigQuery database</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/airflow/contrib/operators/bigquery_to_gcs.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/bigquery_to_gcs.html b/_modules/airflow/contrib/operators/bigquery_to_gcs.html
index a3b8d18..97cf815 100644
--- a/_modules/airflow/contrib/operators/bigquery_to_gcs.html
+++ b/_modules/airflow/contrib/operators/bigquery_to_gcs.html
@@ -157,6 +157,7 @@
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 
+
 <div class="viewcode-block" id="BigQueryToCloudStorageOperator"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.bigquery_to_gcs.BigQueryToCloudStorageOperator">[docs]</a><span class="k">class</span> <span class="nc">BigQueryToCloudStorageOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
     <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Transfers a BigQuery table to a Google Cloud Storage bucket.</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/airflow/contrib/operators/gcs_download_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/gcs_download_operator.html b/_modules/airflow/contrib/operators/gcs_download_operator.html
index dbf56fb..39186e2 100644
--- a/_modules/airflow/contrib/operators/gcs_download_operator.html
+++ b/_modules/airflow/contrib/operators/gcs_download_operator.html
@@ -158,6 +158,7 @@
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 
+
 <div class="viewcode-block" id="GoogleCloudStorageDownloadOperator"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.gcs_download_operator.GoogleCloudStorageDownloadOperator">[docs]</a><span class="k">class</span> <span class="nc">GoogleCloudStorageDownloadOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
     <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Downloads a file from Google Cloud Storage.</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/airflow/executors/local_executor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/executors/local_executor.html b/_modules/airflow/executors/local_executor.html
index 1841cb4..250eafd 100644
--- a/_modules/airflow/executors/local_executor.html
+++ b/_modules/airflow/executors/local_executor.html
@@ -223,7 +223,9 @@
 
     <span class="k">def</span> <span class="nf">end</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
         <span class="c1"># Sending poison pill to all worker</span>
-        <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="bp">None</span><span class="p">,</span> <span class="bp">None</span><span class="p">))</span> <span class="k">for</span> <span class="n">w</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">workers</span><span class="p">]</span>
+        <span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">workers</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="bp">None</span><span class="p">,</span> <span class="bp">None</span><span class="p">))</span>
+
         <span class="c1"># Wait for commands to finish</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">queue</span><span class="o">.</span><span class="n">join</span><span class="p">()</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">sync</span><span class="p">()</span></div>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/airflow/macros.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/macros.html b/_modules/airflow/macros.html
index 78efd22..19c18ff 100644
--- a/_modules/airflow/macros.html
+++ b/_modules/airflow/macros.html
@@ -209,7 +209,6 @@
         <span class="n">sys</span><span class="o">.</span><span class="n">modules</span><span class="p">[</span><span class="n">_macro_module</span><span class="o">.</span><span class="n">__name__</span><span class="p">]</span> <span class="o">=</span> <span class="n">_macro_module</span>
         <span class="nb">globals</span><span class="p">()[</span><span class="n">_macro_module</span><span class="o">.</span><span class="n">_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">_macro_module</span>
 
-
         <span class="c1">##########################################################</span>
         <span class="c1"># TODO FIXME Remove in Airflow 2.0</span>
 

[5/7] incubator-airflow-site git commit: New version of the docs

[ma...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/airflow/operators/sensors.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/sensors.html b/_modules/airflow/operators/sensors.html
index 65a0587..3b716d3 100644
--- a/_modules/airflow/operators/sensors.html
+++ b/_modules/airflow/operators/sensors.html
@@ -155,6 +155,8 @@
 <span class="kn">from</span> <span class="nn">future</span> <span class="kn">import</span> <span class="n">standard_library</span>
 <span class="n">standard_library</span><span class="o">.</span><span class="n">install_aliases</span><span class="p">()</span>
 <span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
+<span class="kn">from</span> <span class="nn">past.builtins</span> <span class="kn">import</span> <span class="nb">basestring</span>
+
 <span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">datetime</span>
 <span class="kn">import</span> <span class="nn">logging</span>
 <span class="kn">from</span> <span class="nn">urllib.parse</span> <span class="kn">import</span> <span class="n">urlparse</span>
@@ -229,6 +231,7 @@
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;sql&#39;</span><span class="p">,)</span>
     <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;.hql&#39;</span><span class="p">,</span> <span class="s1">&#39;.sql&#39;</span><span class="p">,)</span>
+    <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#7c7287&#39;</span>
 
     <span class="nd">@apply_defaults</span>
     <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">conn_id</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
@@ -272,6 +275,7 @@
 <span class="sd">    :type mysql_conn_id: str</span>
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;partition_name&#39;</span><span class="p">,</span> <span class="s1">&#39;table&#39;</span><span class="p">,</span> <span class="s1">&#39;schema&#39;</span><span class="p">)</span>
+    <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#8da7be&#39;</span>
 
     <span class="nd">@apply_defaults</span>
     <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
@@ -327,6 +331,7 @@
 <span class="sd">        or execution_date_fn can be passed to ExternalTaskSensor, but not both.</span>
 <span class="sd">    :type execution_date_fn: callable</span>
 <span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#19647e&#39;</span>
 
     <span class="nd">@apply_defaults</span>
     <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
@@ -382,10 +387,11 @@
 
 <span class="sd">    :param partition_names: List of fully qualified names of the</span>
 <span class="sd">        partitions to wait for. A fully qualified name is of the</span>
-<span class="sd">        form schema.table/pk1=pv1/pk2=pv2, for example,</span>
+<span class="sd">        form ``schema.table/pk1=pv1/pk2=pv2``, for example,</span>
 <span class="sd">        default.users/ds=2016-01-01. This is passed as is to the metastore</span>
-<span class="sd">        Thrift client &quot;get_partitions_by_name&quot; method. Note that</span>
-<span class="sd">        you cannot use logical operators as in HivePartitionSensor.</span>
+<span class="sd">        Thrift client ``get_partitions_by_name`` method. Note that</span>
+<span class="sd">        you cannot use logical or comparison operators as in</span>
+<span class="sd">        HivePartitionSensor.</span>
 <span class="sd">    :type partition_names: list of strings</span>
 <span class="sd">    :param metastore_conn_id: reference to the metastore thrift service</span>
 <span class="sd">        connection id</span>
@@ -393,6 +399,7 @@
 <span class="sd">    &quot;&quot;&quot;</span>
 
     <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;partition_names&#39;</span><span class="p">,</span> <span class="p">)</span>
+    <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#8d99ae&#39;</span>
 
     <span class="nd">@apply_defaults</span>
     <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
@@ -408,9 +415,6 @@
         <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">partition_names</span><span class="p">,</span> <span class="nb">basestring</span><span class="p">):</span>
             <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s1">&#39;partition_names must be an array of strings&#39;</span><span class="p">)</span>
 
-        <span class="k">for</span> <span class="n">partition_name</span> <span class="ow">in</span> <span class="n">partition_names</span><span class="p">:</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">parse_partition_name</span><span class="p">(</span><span class="n">partition_name</span><span class="p">)</span>
-
         <span class="bp">self</span><span class="o">.</span><span class="n">metastore_conn_id</span> <span class="o">=</span> <span class="n">metastore_conn_id</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">partition_names</span> <span class="o">=</span> <span class="n">partition_names</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">next_poke_idx</span> <span class="o">=</span> <span class="mi">0</span>
@@ -452,7 +456,7 @@
     <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Waits for a partition to show up in Hive.</span>
 
-<span class="sd">    Note: Because @partition supports general logical operators, it</span>
+<span class="sd">    Note: Because ``partition`` supports general logical operators, it</span>
 <span class="sd">    can be inefficient. Consider using NamedHivePartitionSensor instead if</span>
 <span class="sd">    you don&#39;t need the full flexibility of HivePartitionSensor.</span>
 
@@ -460,15 +464,16 @@
 <span class="sd">        notation (my_database.my_table)</span>
 <span class="sd">    :type table: string</span>
 <span class="sd">    :param partition: The partition clause to wait for. This is passed as</span>
-<span class="sd">        is to the metastore Thrift client &quot;get_partitions_by_filter&quot; method,</span>
-<span class="sd">        and apparently supports SQL like notation as in `ds=&#39;2015-01-01&#39;</span>
-<span class="sd">        AND type=&#39;value&#39;` and &gt; &lt; sings as in &quot;ds&gt;=2015-01-01&quot;</span>
+<span class="sd">        is to the metastore Thrift client ``get_partitions_by_filter`` method,</span>
+<span class="sd">        and apparently supports SQL like notation as in ``ds=&#39;2015-01-01&#39;</span>
+<span class="sd">        AND type=&#39;value&#39;`` and comparison operators as in ``&quot;ds&gt;=2015-01-01&quot;``</span>
 <span class="sd">    :type partition: string</span>
 <span class="sd">    :param metastore_conn_id: reference to the metastore thrift service</span>
 <span class="sd">        connection id</span>
 <span class="sd">    :type metastore_conn_id: str</span>
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;schema&#39;</span><span class="p">,</span> <span class="s1">&#39;table&#39;</span><span class="p">,</span> <span class="s1">&#39;partition&#39;</span><span class="p">,)</span>
+    <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#2b2d42&#39;</span>
 
     <span class="nd">@apply_defaults</span>
     <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
@@ -505,6 +510,7 @@
 <span class="sd">    Waits for a file or folder to land in HDFS</span>
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;filepath&#39;</span><span class="p">,)</span>
+    <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#4d9de0&#39;</span>
 
     <span class="nd">@apply_defaults</span>
     <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/bash_operator.html
----------------------------------------------------------------------
diff --git a/_modules/bash_operator.html b/_modules/bash_operator.html
index d38a3f8..9877b4a 100644
--- a/_modules/bash_operator.html
+++ b/_modules/bash_operator.html
@@ -170,6 +170,9 @@
 <span class="sd">    :param bash_command: The command, set of commands or reference to a</span>
 <span class="sd">        bash script (must be &#39;.sh&#39;) to be executed.</span>
 <span class="sd">    :type bash_command: string</span>
+<span class="sd">    :param xcom_push: If xcom_push is True, the last line written to stdout</span>
+<span class="sd">        will also be pushed to an XCom when the bash command completes.</span>
+<span class="sd">    :type xcom_push: bool</span>
 <span class="sd">    :param env: If env is not None, it must be a mapping that defines the</span>
 <span class="sd">        environment variables for the new process; these are used instead</span>
 <span class="sd">        of inheriting the current process environment, which is the default</span>
@@ -189,10 +192,7 @@
             <span class="n">env</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
             <span class="n">output_encoding</span><span class="o">=</span><span class="s1">&#39;utf-8&#39;</span><span class="p">,</span>
             <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
-        <span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">        If xcom_push is True, the last line written to stdout will also</span>
-<span class="sd">        be pushed to an XCom when the bash command completes.</span>
-<span class="sd">        &quot;&quot;&quot;</span>
+
         <span class="nb">super</span><span class="p">(</span><span class="n">BashOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">bash_command</span> <span class="o">=</span> <span class="n">bash_command</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">env</span> <span class="o">=</span> <span class="n">env</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/bigquery_hook.html
----------------------------------------------------------------------
diff --git a/_modules/bigquery_hook.html b/_modules/bigquery_hook.html
index 9478587..1f8170a 100644
--- a/_modules/bigquery_hook.html
+++ b/_modules/bigquery_hook.html
@@ -157,6 +157,9 @@
 <span class="sd">implementation for BigQuery.</span>
 <span class="sd">&quot;&quot;&quot;</span>
 
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">range</span>
+<span class="kn">from</span> <span class="nn">past.builtins</span> <span class="kn">import</span> <span class="nb">basestring</span>
+
 <span class="kn">import</span> <span class="nn">logging</span>
 <span class="kn">import</span> <span class="nn">time</span>
 
@@ -440,10 +443,10 @@
                 <span class="bp">self</span><span class="o">.</span><span class="n">_split_project_dataset_table_input</span><span class="p">(</span>
                     <span class="s1">&#39;source_project_dataset_table&#39;</span><span class="p">,</span> <span class="n">source_project_dataset_table</span><span class="p">)</span>
             <span class="n">source_project_dataset_tables_fixup</span><span class="o">.</span><span class="n">append</span><span class="p">({</span>
-                    <span class="s1">&#39;projectId&#39;</span><span class="p">:</span> <span class="n">source_project</span><span class="p">,</span>
-                    <span class="s1">&#39;datasetId&#39;</span><span class="p">:</span> <span class="n">source_dataset</span><span class="p">,</span>
-                    <span class="s1">&#39;tableId&#39;</span><span class="p">:</span> <span class="n">source_table</span>
-                <span class="p">})</span>
+                <span class="s1">&#39;projectId&#39;</span><span class="p">:</span> <span class="n">source_project</span><span class="p">,</span>
+                <span class="s1">&#39;datasetId&#39;</span><span class="p">:</span> <span class="n">source_dataset</span><span class="p">,</span>
+                <span class="s1">&#39;tableId&#39;</span><span class="p">:</span> <span class="n">source_table</span>
+            <span class="p">})</span>
 
         <span class="k">assert</span> <span class="mi">3</span> <span class="o">==</span> <span class="nb">len</span><span class="p">(</span><span class="n">destination_project_dataset_table</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">)),</span> <span class="p">(</span>
             <span class="s1">&#39;Expected destination_project_dataset_table in the format of &#39;</span>
@@ -652,29 +655,34 @@
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="c1"># check to see if the table exists</span>
         <span class="n">table_id</span> <span class="o">=</span> <span class="n">table_resource</span><span class="p">[</span><span class="s1">&#39;tableReference&#39;</span><span class="p">][</span><span class="s1">&#39;tableId&#39;</span><span class="p">]</span>
-        <span class="n">table_exists</span> <span class="o">=</span> <span class="bp">False</span>
         <span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span> <span class="k">if</span> <span class="n">project_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span>
         <span class="n">tables_list_resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">list</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
                                                       <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
-        <span class="k">if</span> <span class="s1">&#39;tables&#39;</span> <span class="ow">in</span> <span class="n">tables_list_resp</span><span class="p">:</span>
-            <span class="k">for</span> <span class="n">table</span> <span class="ow">in</span> <span class="n">tables_list_resp</span><span class="p">[</span><span class="s1">&#39;tables&#39;</span><span class="p">]:</span>
+        <span class="k">while</span> <span class="bp">True</span><span class="p">:</span>
+            <span class="k">for</span> <span class="n">table</span> <span class="ow">in</span> <span class="n">tables_list_resp</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;tables&#39;</span><span class="p">,</span> <span class="p">[]):</span>
                 <span class="k">if</span> <span class="n">table</span><span class="p">[</span><span class="s1">&#39;tableReference&#39;</span><span class="p">][</span><span class="s1">&#39;tableId&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="n">table_id</span><span class="p">:</span>
-                    <span class="n">table_exists</span> <span class="o">=</span> <span class="bp">True</span>
-                    <span class="k">break</span>
-
-        <span class="c1"># do update if table exists</span>
-        <span class="k">if</span> <span class="n">table_exists</span><span class="p">:</span>
-            <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1"> exists, updating.&#39;</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">)</span>
-            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
-                                                <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
-                                                <span class="n">tableId</span><span class="o">=</span><span class="n">table_id</span><span class="p">,</span>
-                                                <span class="n">body</span><span class="o">=</span><span class="n">table_resource</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
-        <span class="c1"># do insert if table does not exist</span>
-        <span class="k">else</span><span class="p">:</span>
-            <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1"> does not exist. creating.&#39;</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">)</span>
-            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">insert</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
-                                                <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
-                                                <span class="n">body</span><span class="o">=</span><span class="n">table_resource</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+                    <span class="c1"># found the table, do update</span>
+                    <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1"> exists, updating.&#39;</span><span class="p">,</span>
+                                 <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">)</span>
+                    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
+                                                        <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
+                                                        <span class="n">tableId</span><span class="o">=</span><span class="n">table_id</span><span class="p">,</span>
+                                                        <span class="n">body</span><span class="o">=</span><span class="n">table_resource</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+            <span class="c1"># If there is a next page, we need to check the next page.</span>
+            <span class="k">if</span> <span class="s1">&#39;nextPageToken&#39;</span> <span class="ow">in</span> <span class="n">tables_list_resp</span><span class="p">:</span>
+                <span class="n">tables_list_resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span>\
+                    <span class="o">.</span><span class="n">list</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
+                          <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
+                          <span class="n">pageToken</span><span class="o">=</span><span class="n">tables_list_resp</span><span class="p">[</span><span class="s1">&#39;nextPageToken&#39;</span><span class="p">])</span>\
+                    <span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+            <span class="c1"># If there is no next page, then the table doesn&#39;t exist.</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="c1"># do insert</span>
+                <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1"> does not exist. creating.&#39;</span><span class="p">,</span>
+                             <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">)</span>
+                <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">insert</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
+                                                    <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
+                                                    <span class="n">body</span><span class="o">=</span><span class="n">table_resource</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
 
     <span class="k">def</span> <span class="nf">run_grant_dataset_view_access</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
                                       <span class="n">source_dataset</span><span class="p">,</span>
@@ -851,7 +859,7 @@
         <span class="k">if</span> <span class="n">size</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
             <span class="n">size</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">arraysize</span>
         <span class="n">result</span> <span class="o">=</span> <span class="p">[]</span>
-        <span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="nb">xrange</span><span class="p">(</span><span class="n">size</span><span class="p">):</span>
+        <span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">size</span><span class="p">):</span>
             <span class="n">one</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">fetchone</span><span class="p">()</span>
             <span class="k">if</span> <span class="n">one</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
                 <span class="k">break</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/cloudant_hook.html
----------------------------------------------------------------------
diff --git a/_modules/cloudant_hook.html b/_modules/cloudant_hook.html
index b238080..b7747b4 100644
--- a/_modules/cloudant_hook.html
+++ b/_modules/cloudant_hook.html
@@ -150,9 +150,11 @@
 <span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
 <span class="c1"># See the License for the specific language governing permissions and</span>
 <span class="c1"># limitations under the License.</span>
-<span class="kn">import</span> <span class="nn">logging</span>
+
+<span class="kn">from</span> <span class="nn">past.builtins</span> <span class="kn">import</span> <span class="nb">unicode</span>
 
 <span class="kn">import</span> <span class="nn">cloudant</span>
+<span class="kn">import</span> <span class="nn">logging</span>
 
 <span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span>
 <span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="kn">import</span> <span class="n">BaseHook</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/dagrun_operator.html
----------------------------------------------------------------------
diff --git a/_modules/dagrun_operator.html b/_modules/dagrun_operator.html
index c6725bb..1d147e0 100644
--- a/_modules/dagrun_operator.html
+++ b/_modules/dagrun_operator.html
@@ -185,6 +185,7 @@
     <span class="n">template_fields</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">()</span>
     <span class="n">template_ext</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">()</span>
     <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#ffefeb&#39;</span>
+
     <span class="nd">@apply_defaults</span>
     <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
             <span class="bp">self</span><span class="p">,</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/dbapi_hook.html
----------------------------------------------------------------------
diff --git a/_modules/dbapi_hook.html b/_modules/dbapi_hook.html
index e608a23..604dddd 100644
--- a/_modules/dbapi_hook.html
+++ b/_modules/dbapi_hook.html
@@ -195,9 +195,8 @@
             <span class="n">username</span><span class="o">=</span><span class="n">db</span><span class="o">.</span><span class="n">login</span><span class="p">,</span>
             <span class="n">schema</span><span class="o">=</span><span class="n">db</span><span class="o">.</span><span class="n">schema</span><span class="p">)</span></div>
 
-
 <div class="viewcode-block" id="DbApiHook.get_pandas_df"><a class="viewcode-back" href="../code.html#airflow.hooks.DbApiHook.get_pandas_df">[docs]</a>    <span class="k">def</span> <span class="nf">get_pandas_df</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
-        <span class="sd">&#39;&#39;&#39;</span>
+        <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Executes the sql and returns a pandas dataframe</span>
 
 <span class="sd">        :param sql: the sql statement to be executed (str) or a list of</span>
@@ -205,7 +204,7 @@
 <span class="sd">        :type sql: str or list</span>
 <span class="sd">        :param parameters: The parameters to render the SQL query with.</span>
 <span class="sd">        :type parameters: mapping or iterable</span>
-<span class="sd">        &#39;&#39;&#39;</span>
+<span class="sd">        &quot;&quot;&quot;</span>
         <span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">version_info</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">&lt;</span> <span class="mi">3</span><span class="p">:</span>
             <span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span>
         <span class="kn">import</span> <span class="nn">pandas.io.sql</span> <span class="kn">as</span> <span class="nn">psql</span>
@@ -215,7 +214,7 @@
         <span class="k">return</span> <span class="n">df</span></div>
 
 <div class="viewcode-block" id="DbApiHook.get_records"><a class="viewcode-back" href="../code.html#airflow.hooks.DbApiHook.get_records">[docs]</a>    <span class="k">def</span> <span class="nf">get_records</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
-        <span class="sd">&#39;&#39;&#39;</span>
+        <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Executes the sql and returns a set of records.</span>
 
 <span class="sd">        :param sql: the sql statement to be executed (str) or a list of</span>
@@ -223,7 +222,7 @@
 <span class="sd">        :type sql: str or list</span>
 <span class="sd">        :param parameters: The parameters to render the SQL query with.</span>
 <span class="sd">        :type parameters: mapping or iterable</span>
-<span class="sd">        &#39;&#39;&#39;</span>
+<span class="sd">        &quot;&quot;&quot;</span>
         <span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">version_info</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">&lt;</span> <span class="mi">3</span><span class="p">:</span>
             <span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span>
         <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
@@ -238,7 +237,7 @@
         <span class="k">return</span> <span class="n">rows</span></div>
 
 <div class="viewcode-block" id="DbApiHook.get_first"><a class="viewcode-back" href="../code.html#airflow.hooks.DbApiHook.get_first">[docs]</a>    <span class="k">def</span> <span class="nf">get_first</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
-        <span class="sd">&#39;&#39;&#39;</span>
+        <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Executes the sql and returns the first resulting row.</span>
 
 <span class="sd">        :param sql: the sql statement to be executed (str) or a list of</span>
@@ -246,7 +245,7 @@
 <span class="sd">        :type sql: str or list</span>
 <span class="sd">        :param parameters: The parameters to render the SQL query with.</span>
 <span class="sd">        :type parameters: mapping or iterable</span>
-<span class="sd">        &#39;&#39;&#39;</span>
+<span class="sd">        &quot;&quot;&quot;</span>
         <span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">version_info</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">&lt;</span> <span class="mi">3</span><span class="p">:</span>
             <span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span>
         <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
@@ -280,7 +279,7 @@
             <span class="n">sql</span> <span class="o">=</span> <span class="p">[</span><span class="n">sql</span><span class="p">]</span>
 
         <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">supports_autocommit</span><span class="p">:</span>
-           <span class="bp">self</span><span class="o">.</span><span class="n">set_autocommit</span><span class="p">(</span><span class="n">conn</span><span class="p">,</span> <span class="n">autocommit</span><span class="p">)</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">set_autocommit</span><span class="p">(</span><span class="n">conn</span><span class="p">,</span> <span class="n">autocommit</span><span class="p">)</span>
 
         <span class="n">cur</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span>
         <span class="k">for</span> <span class="n">s</span> <span class="ow">in</span> <span class="n">sql</span><span class="p">:</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/druid_hook.html
----------------------------------------------------------------------
diff --git a/_modules/druid_hook.html b/_modules/druid_hook.html
index f5b2a4d..20ffb1d 100644
--- a/_modules/druid_hook.html
+++ b/_modules/druid_hook.html
@@ -165,6 +165,7 @@
 <span class="n">LOAD_CHECK_INTERVAL</span> <span class="o">=</span> <span class="mi">5</span>
 <span class="n">DEFAULT_TARGET_PARTITION_SIZE</span> <span class="o">=</span> <span class="mi">5000000</span>
 
+
 <span class="k">class</span> <span class="nc">AirflowDruidLoadException</span><span class="p">(</span><span class="n">AirflowException</span><span class="p">):</span>
     <span class="k">pass</span>
 
@@ -260,10 +261,10 @@
                         <span class="s2">&quot;mapreduce.map.output.compress&quot;</span><span class="p">:</span> <span class="s2">&quot;false&quot;</span><span class="p">,</span>
                         <span class="s2">&quot;mapreduce.output.fileoutputformat.compress&quot;</span><span class="p">:</span> <span class="s2">&quot;false&quot;</span><span class="p">,</span>
                     <span class="p">},</span>
-                    <span class="s2">&quot;partitionsSpec&quot;</span> <span class="p">:</span> <span class="p">{</span>
-                        <span class="s2">&quot;type&quot;</span> <span class="p">:</span> <span class="s2">&quot;hashed&quot;</span><span class="p">,</span>
-                        <span class="s2">&quot;targetPartitionSize&quot;</span> <span class="p">:</span> <span class="n">target_partition_size</span><span class="p">,</span>
-                        <span class="s2">&quot;numShards&quot;</span> <span class="p">:</span> <span class="n">num_shards</span><span class="p">,</span>
+                    <span class="s2">&quot;partitionsSpec&quot;</span><span class="p">:</span> <span class="p">{</span>
+                        <span class="s2">&quot;type&quot;</span><span class="p">:</span> <span class="s2">&quot;hashed&quot;</span><span class="p">,</span>
+                        <span class="s2">&quot;targetPartitionSize&quot;</span><span class="p">:</span> <span class="n">target_partition_size</span><span class="p">,</span>
+                        <span class="s2">&quot;numShards&quot;</span><span class="p">:</span> <span class="n">num_shards</span><span class="p">,</span>
                     <span class="p">},</span>
                 <span class="p">},</span>
                 <span class="s2">&quot;ioConfig&quot;</span><span class="p">:</span> <span class="p">{</span>
@@ -303,8 +304,9 @@
             <span class="n">intervals</span><span class="p">,</span> <span class="n">num_shards</span><span class="p">,</span> <span class="n">target_partition_size</span><span class="p">,</span> <span class="n">metric_spec</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">hadoop_dependency_coordinates</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        load data to druid from hdfs</span>
-<span class="sd">        :params ts_dim: The column name to use as a timestamp</span>
-<span class="sd">        :params metric_spec: A list of dictionaries</span>
+
+<span class="sd">        :param ts_dim: The column name to use as a timestamp</span>
+<span class="sd">        :param metric_spec: A list of dictionaries</span>
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="n">task_id</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">send_ingest_query</span><span class="p">(</span>
             <span class="n">datasource</span><span class="p">,</span> <span class="n">static_path</span><span class="p">,</span> <span class="n">ts_dim</span><span class="p">,</span> <span class="n">columns</span><span class="p">,</span> <span class="n">metric_spec</span><span class="p">,</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/email_operator.html
----------------------------------------------------------------------
diff --git a/_modules/email_operator.html b/_modules/email_operator.html
index 6bb4434..4d91901 100644
--- a/_modules/email_operator.html
+++ b/_modules/email_operator.html
@@ -169,6 +169,10 @@
 <span class="sd">    :type html_content: string</span>
 <span class="sd">    :param files: file names to attach in email</span>
 <span class="sd">    :type files: list</span>
+<span class="sd">    :param cc: list of recipients to be added in CC field</span>
+<span class="sd">    :type cc: list or string (comma or semicolon delimited)</span>
+<span class="sd">    :param bcc: list of recipients to be added in BCC field</span>
+<span class="sd">    :type bcc: list or string (comma or semicolon delimited)</span>
 <span class="sd">    &quot;&quot;&quot;</span>
 
     <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;subject&#39;</span><span class="p">,</span> <span class="s1">&#39;html_content&#39;</span><span class="p">)</span>
@@ -182,15 +186,19 @@
             <span class="n">subject</span><span class="p">,</span>
             <span class="n">html_content</span><span class="p">,</span>
             <span class="n">files</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+            <span class="n">cc</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+            <span class="n">bcc</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
             <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
         <span class="nb">super</span><span class="p">(</span><span class="n">EmailOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">to</span> <span class="o">=</span> <span class="n">to</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">subject</span> <span class="o">=</span> <span class="n">subject</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">html_content</span> <span class="o">=</span> <span class="n">html_content</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">files</span> <span class="o">=</span> <span class="n">files</span> <span class="ow">or</span> <span class="p">[]</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">cc</span> <span class="o">=</span> <span class="n">cc</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">bcc</span> <span class="o">=</span> <span class="n">bcc</span>
 
     <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
-        <span class="n">send_email</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">to</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">subject</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">html_content</span><span class="p">,</span> <span class="n">files</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">files</span><span class="p">)</span></div>
+        <span class="n">send_email</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">to</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">subject</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">html_content</span><span class="p">,</span> <span class="n">files</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">files</span><span class="p">,</span> <span class="n">cc</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">cc</span><span class="p">,</span> <span class="n">bcc</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bcc</span><span class="p">)</span></div>
 </pre></div>
 
            </div>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/ftp_hook.html
----------------------------------------------------------------------
diff --git a/_modules/ftp_hook.html b/_modules/ftp_hook.html
index 19a7498..eed5b97 100644
--- a/_modules/ftp_hook.html
+++ b/_modules/ftp_hook.html
@@ -352,6 +352,16 @@
         <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
         <span class="n">conn</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span><span class="n">path</span><span class="p">)</span></div>
 
+<div class="viewcode-block" id="FTPHook.rename"><a class="viewcode-back" href="../code.html#airflow.contrib.hooks.FTPHook.rename">[docs]</a>    <span class="k">def</span> <span class="nf">rename</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">from_name</span><span class="p">,</span> <span class="n">to_name</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Rename a file.</span>
+
+<span class="sd">        :param from_name: rename file from name</span>
+<span class="sd">        :param to_name: rename file to name</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+        <span class="k">return</span> <span class="n">conn</span><span class="o">.</span><span class="n">rename</span><span class="p">(</span><span class="n">from_name</span><span class="p">,</span> <span class="n">to_name</span><span class="p">)</span></div>
+
     <span class="k">def</span> <span class="nf">get_mod_time</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
         <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
         <span class="n">ftp_mdtm</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">sendcmd</span><span class="p">(</span><span class="s1">&#39;MDTM &#39;</span> <span class="o">+</span> <span class="n">path</span><span class="p">)</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/hive_hooks.html
----------------------------------------------------------------------
diff --git a/_modules/hive_hooks.html b/_modules/hive_hooks.html
index e25ec97..926b54c 100644
--- a/_modules/hive_hooks.html
+++ b/_modules/hive_hooks.html
@@ -156,19 +156,25 @@
 <span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">zip</span>
 <span class="kn">from</span> <span class="nn">past.builtins</span> <span class="kn">import</span> <span class="nb">basestring</span>
 
+<span class="kn">import</span> <span class="nn">collections</span>
 <span class="kn">import</span> <span class="nn">unicodecsv</span> <span class="kn">as</span> <span class="nn">csv</span>
+<span class="kn">import</span> <span class="nn">itertools</span>
 <span class="kn">import</span> <span class="nn">logging</span>
 <span class="kn">import</span> <span class="nn">re</span>
 <span class="kn">import</span> <span class="nn">subprocess</span>
+<span class="kn">import</span> <span class="nn">time</span>
 <span class="kn">from</span> <span class="nn">tempfile</span> <span class="kn">import</span> <span class="n">NamedTemporaryFile</span>
 <span class="kn">import</span> <span class="nn">hive_metastore</span>
 
 <span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span>
 <span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="kn">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.helpers</span> <span class="kn">import</span> <span class="n">as_flattened_list</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.file</span> <span class="kn">import</span> <span class="n">TemporaryDirectory</span>
 <span class="kn">from</span> <span class="nn">airflow</span> <span class="kn">import</span> <span class="n">configuration</span>
 <span class="kn">import</span> <span class="nn">airflow.security.utils</span> <span class="kn">as</span> <span class="nn">utils</span>
 
+<span class="n">HIVE_QUEUE_PRIORITIES</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;VERY_HIGH&#39;</span><span class="p">,</span> <span class="s1">&#39;HIGH&#39;</span><span class="p">,</span> <span class="s1">&#39;NORMAL&#39;</span><span class="p">,</span> <span class="s1">&#39;LOW&#39;</span><span class="p">,</span> <span class="s1">&#39;VERY_LOW&#39;</span><span class="p">]</span>
+
 
 <div class="viewcode-block" id="HiveCliHook"><a class="viewcode-back" href="../code.html#airflow.hooks.HiveCliHook">[docs]</a><span class="k">class</span> <span class="nc">HiveCliHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span>
 
@@ -186,12 +192,24 @@
 
 <span class="sd">    The extra connection parameter ``auth`` gets passed as in the ``jdbc``</span>
 <span class="sd">    connection string as is.</span>
+
+<span class="sd">    :param mapred_queue: queue used by the Hadoop Scheduler (Capacity or Fair)</span>
+<span class="sd">    :type  mapred_queue: string</span>
+<span class="sd">    :param mapred_queue_priority: priority within the job queue.</span>
+<span class="sd">        Possible settings include: VERY_HIGH, HIGH, NORMAL, LOW, VERY_LOW</span>
+<span class="sd">    :type  mapred_queue_priority: string</span>
+<span class="sd">    :param mapred_job_name: This name will appear in the jobtracker.</span>
+<span class="sd">        This can make monitoring easier.</span>
+<span class="sd">    :type  mapred_job_name: string</span>
 <span class="sd">    &quot;&quot;&quot;</span>
 
     <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
             <span class="bp">self</span><span class="p">,</span>
             <span class="n">hive_cli_conn_id</span><span class="o">=</span><span class="s2">&quot;hive_cli_default&quot;</span><span class="p">,</span>
-            <span class="n">run_as</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
+            <span class="n">run_as</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+            <span class="n">mapred_queue</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+            <span class="n">mapred_queue_priority</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+            <span class="n">mapred_job_name</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
         <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="n">hive_cli_conn_id</span><span class="p">)</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">hive_cli_params</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;hive_cli_params&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">use_beeline</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;use_beeline&#39;</span><span class="p">,</span> <span class="bp">False</span><span class="p">)</span>
@@ -199,16 +217,92 @@
         <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="n">conn</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">run_as</span> <span class="o">=</span> <span class="n">run_as</span>
 
+        <span class="k">if</span> <span class="n">mapred_queue_priority</span><span class="p">:</span>
+            <span class="n">mapred_queue_priority</span> <span class="o">=</span> <span class="n">mapred_queue_priority</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span>
+            <span class="k">if</span> <span class="n">mapred_queue_priority</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">HIVE_QUEUE_PRIORITIES</span><span class="p">:</span>
+                <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
+                    <span class="s2">&quot;Invalid Mapred Queue Priority.  Valid values are: &quot;</span>
+                    <span class="s2">&quot;{}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="s1">&#39;, &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">HIVE_QUEUE_PRIORITIES</span><span class="p">)))</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">mapred_queue</span> <span class="o">=</span> <span class="n">mapred_queue</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mapred_queue_priority</span> <span class="o">=</span> <span class="n">mapred_queue_priority</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mapred_job_name</span> <span class="o">=</span> <span class="n">mapred_job_name</span>
+
+    <span class="k">def</span> <span class="nf">_prepare_cli_cmd</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        This function creates the command list from available information</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span>
+        <span class="n">hive_bin</span> <span class="o">=</span> <span class="s1">&#39;hive&#39;</span>
+        <span class="n">cmd_extra</span> <span class="o">=</span> <span class="p">[]</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_beeline</span><span class="p">:</span>
+            <span class="n">hive_bin</span> <span class="o">=</span> <span class="s1">&#39;beeline&#39;</span>
+            <span class="n">jdbc_url</span> <span class="o">=</span> <span class="s2">&quot;jdbc:hive2://{conn.host}:{conn.port}/{conn.schema}&quot;</span>
+            <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;core&#39;</span><span class="p">,</span> <span class="s1">&#39;security&#39;</span><span class="p">)</span> <span class="o">==</span> <span class="s1">&#39;kerberos&#39;</span><span class="p">:</span>
+                <span class="n">template</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+                    <span class="s1">&#39;principal&#39;</span><span class="p">,</span> <span class="s2">&quot;hive/_HOST@EXAMPLE.COM&quot;</span><span class="p">)</span>
+                <span class="k">if</span> <span class="s2">&quot;_HOST&quot;</span> <span class="ow">in</span> <span class="n">template</span><span class="p">:</span>
+                    <span class="n">template</span> <span class="o">=</span> <span class="n">utils</span><span class="o">.</span><span class="n">replace_hostname_pattern</span><span class="p">(</span>
+                        <span class="n">utils</span><span class="o">.</span><span class="n">get_components</span><span class="p">(</span><span class="n">template</span><span class="p">))</span>
+
+                <span class="n">proxy_user</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>  <span class="c1"># noqa</span>
+                <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;proxy_user&#39;</span><span class="p">)</span> <span class="o">==</span> <span class="s2">&quot;login&quot;</span> <span class="ow">and</span> <span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">:</span>
+                    <span class="n">proxy_user</span> <span class="o">=</span> <span class="s2">&quot;hive.server2.proxy.user={0}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">)</span>
+                <span class="k">elif</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;proxy_user&#39;</span><span class="p">)</span> <span class="o">==</span> <span class="s2">&quot;owner&quot;</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_as</span><span class="p">:</span>
+                    <span class="n">proxy_user</span> <span class="o">=</span> <span class="s2">&quot;hive.server2.proxy.user={0}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">run_as</span><span class="p">)</span>
+
+                <span class="n">jdbc_url</span> <span class="o">+=</span> <span class="s2">&quot;;principal={template};{proxy_user}&quot;</span>
+            <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">auth</span><span class="p">:</span>
+                <span class="n">jdbc_url</span> <span class="o">+=</span> <span class="s2">&quot;;auth=&quot;</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">auth</span>
+
+            <span class="n">jdbc_url</span> <span class="o">=</span> <span class="n">jdbc_url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">())</span>
+
+            <span class="n">cmd_extra</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">&#39;-u&#39;</span><span class="p">,</span> <span class="n">jdbc_url</span><span class="p">]</span>
+            <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">:</span>
+                <span class="n">cmd_extra</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">&#39;-n&#39;</span><span class="p">,</span> <span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">]</span>
+            <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">:</span>
+                <span class="n">cmd_extra</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">&#39;-p&#39;</span><span class="p">,</span> <span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">]</span>
+
+        <span class="n">hive_params_list</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">hive_cli_params</span><span class="o">.</span><span class="n">split</span><span class="p">()</span>
+
+        <span class="k">return</span> <span class="p">[</span><span class="n">hive_bin</span><span class="p">]</span> <span class="o">+</span> <span class="n">cmd_extra</span> <span class="o">+</span> <span class="n">hive_params_list</span>
+
+    <span class="k">def</span> <span class="nf">_prepare_hiveconf</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">d</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        This function prepares a list of hiveconf params</span>
+<span class="sd">        from a dictionary of key value pairs.</span>
+
+<span class="sd">        :param d:</span>
+<span class="sd">        :type d: dict</span>
+
+<span class="sd">        &gt;&gt;&gt; hh = HiveCliHook()</span>
+<span class="sd">        &gt;&gt;&gt; hive_conf = {&quot;hive.exec.dynamic.partition&quot;: &quot;true&quot;,</span>
+<span class="sd">        ... &quot;hive.exec.dynamic.partition.mode&quot;: &quot;nonstrict&quot;}</span>
+<span class="sd">        &gt;&gt;&gt; hh._prepare_hiveconf(hive_conf)</span>
+<span class="sd">        [&quot;-hiveconf&quot;, &quot;hive.exec.dynamic.partition=true&quot;,\</span>
+<span class="sd"> &quot;-hiveconf&quot;, &quot;hive.exec.dynamic.partition.mode=nonstrict&quot;]</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">if</span> <span class="ow">not</span> <span class="n">d</span><span class="p">:</span>
+            <span class="k">return</span> <span class="p">[]</span>
+        <span class="k">return</span> <span class="n">as_flattened_list</span><span class="p">(</span>
+            <span class="n">itertools</span><span class="o">.</span><span class="n">izip</span><span class="p">(</span>
+                <span class="p">[</span><span class="s2">&quot;-hiveconf&quot;</span><span class="p">]</span> <span class="o">*</span> <span class="nb">len</span><span class="p">(</span><span class="n">d</span><span class="p">),</span>
+                <span class="p">[</span><span class="s2">&quot;{}={}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">k</span><span class="p">,</span> <span class="n">v</span><span class="p">)</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">d</span><span class="o">.</span><span class="n">items</span><span class="p">()]</span>
+                <span class="p">)</span>
+            <span class="p">)</span>
+
 <div class="viewcode-block" id="HiveCliHook.run_cli"><a class="viewcode-back" href="../code.html#airflow.hooks.HiveCliHook.run_cli">[docs]</a>    <span class="k">def</span> <span class="nf">run_cli</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">hql</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span> <span class="n">hive_conf</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">        Run an hql statement using the hive cli. If hive_conf is specified it should be a</span>
-<span class="sd">        dict and the entries will be set as key/value pairs in HiveConf</span>
+<span class="sd">        Run an hql statement using the hive cli. If hive_conf is specified</span>
+<span class="sd">        it should be a dict and the entries will be set as key/value pairs</span>
+<span class="sd">        in HiveConf</span>
 
 
-<span class="sd">        :param hive_conf: if specified these key value pairs will be passed to hive as</span>
-<span class="sd">            ``-hiveconf &quot;key&quot;=&quot;value&quot;``. Note that they will be passed after the</span>
-<span class="sd">            ``hive_cli_params`` and thus will override whatever values are specified in</span>
-<span class="sd">            the database.</span>
+<span class="sd">        :param hive_conf: if specified these key value pairs will be passed</span>
+<span class="sd">            to hive as ``-hiveconf &quot;key&quot;=&quot;value&quot;``. Note that they will be</span>
+<span class="sd">            passed after the ``hive_cli_params`` and thus will override</span>
+<span class="sd">            whatever values are specified in the database.</span>
 <span class="sd">        :type hive_conf: dict</span>
 
 <span class="sd">        &gt;&gt;&gt; hh = HiveCliHook()</span>
@@ -225,47 +319,29 @@
             <span class="k">with</span> <span class="n">NamedTemporaryFile</span><span class="p">(</span><span class="nb">dir</span><span class="o">=</span><span class="n">tmp_dir</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
                 <span class="n">f</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">hql</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">&#39;UTF-8&#39;</span><span class="p">))</span>
                 <span class="n">f</span><span class="o">.</span><span class="n">flush</span><span class="p">()</span>
-                <span class="n">fname</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">name</span>
-                <span class="n">hive_bin</span> <span class="o">=</span> <span class="s1">&#39;hive&#39;</span>
-                <span class="n">cmd_extra</span> <span class="o">=</span> <span class="p">[]</span>
-
-                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_beeline</span><span class="p">:</span>
-                    <span class="n">hive_bin</span> <span class="o">=</span> <span class="s1">&#39;beeline&#39;</span>
-                    <span class="n">jdbc_url</span> <span class="o">=</span> <span class="s2">&quot;jdbc:hive2://{conn.host}:{conn.port}/{conn.schema}&quot;</span>
-                    <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;core&#39;</span><span class="p">,</span> <span class="s1">&#39;security&#39;</span><span class="p">)</span> <span class="o">==</span> <span class="s1">&#39;kerberos&#39;</span><span class="p">:</span>
-                        <span class="n">template</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
-                            <span class="s1">&#39;principal&#39;</span><span class="p">,</span> <span class="s2">&quot;hive/_HOST@EXAMPLE.COM&quot;</span><span class="p">)</span>
-                        <span class="k">if</span> <span class="s2">&quot;_HOST&quot;</span> <span class="ow">in</span> <span class="n">template</span><span class="p">:</span>
-                            <span class="n">template</span> <span class="o">=</span> <span class="n">utils</span><span class="o">.</span><span class="n">replace_hostname_pattern</span><span class="p">(</span>
-                                <span class="n">utils</span><span class="o">.</span><span class="n">get_components</span><span class="p">(</span><span class="n">template</span><span class="p">))</span>
-
-                        <span class="n">proxy_user</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>  <span class="c1"># noqa</span>
-                        <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;proxy_user&#39;</span><span class="p">)</span> <span class="o">==</span> <span class="s2">&quot;login&quot;</span> <span class="ow">and</span> <span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">:</span>
-                            <span class="n">proxy_user</span> <span class="o">=</span> <span class="s2">&quot;hive.server2.proxy.user={0}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">)</span>
-                        <span class="k">elif</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;proxy_user&#39;</span><span class="p">)</span> <span class="o">==</span> <span class="s2">&quot;owner&quot;</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_as</span><span class="p">:</span>
-                            <span class="n">proxy_user</span> <span class="o">=</span> <span class="s2">&quot;hive.server2.proxy.user={0}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">run_as</span><span class="p">)</span>
-
-                        <span class="n">jdbc_url</span> <span class="o">+=</span> <span class="s2">&quot;;principal={template};{proxy_user}&quot;</span>
-                    <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">auth</span><span class="p">:</span>
-                        <span class="n">jdbc_url</span> <span class="o">+=</span> <span class="s2">&quot;;auth=&quot;</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">auth</span>
-
-                    <span class="n">jdbc_url</span> <span class="o">=</span> <span class="n">jdbc_url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">())</span>
-
-                    <span class="n">cmd_extra</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">&#39;-u&#39;</span><span class="p">,</span> <span class="n">jdbc_url</span><span class="p">]</span>
-                    <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">:</span>
-                        <span class="n">cmd_extra</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">&#39;-n&#39;</span><span class="p">,</span> <span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">]</span>
-                    <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">:</span>
-                        <span class="n">cmd_extra</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">&#39;-p&#39;</span><span class="p">,</span> <span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">]</span>
-
-                <span class="n">hive_conf</span> <span class="o">=</span> <span class="n">hive_conf</span> <span class="ow">or</span> <span class="p">{}</span>
-                <span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">hive_conf</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
-                    <span class="n">cmd_extra</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">&#39;-hiveconf&#39;</span><span class="p">,</span> <span class="s1">&#39;{0}={1}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">value</span><span class="p">)]</span>
-
-                <span class="n">hive_cmd</span> <span class="o">=</span> <span class="p">[</span><span class="n">hive_bin</span><span class="p">,</span> <span class="s1">&#39;-f&#39;</span><span class="p">,</span> <span class="n">fname</span><span class="p">]</span> <span class="o">+</span> <span class="n">cmd_extra</span>
-
-                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">hive_cli_params</span><span class="p">:</span>
-                    <span class="n">hive_params_list</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">hive_cli_params</span><span class="o">.</span><span class="n">split</span><span class="p">()</span>
-                    <span class="n">hive_cmd</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">hive_params_list</span><span class="p">)</span>
+                <span class="n">hive_cmd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prepare_cli_cmd</span><span class="p">()</span>
+                <span class="n">hive_conf_params</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prepare_hiveconf</span><span class="p">(</span><span class="n">hive_conf</span><span class="p">)</span>
+                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mapred_queue</span><span class="p">:</span>
+                    <span class="n">hive_conf_params</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span>
+                        <span class="p">[</span><span class="s1">&#39;-hiveconf&#39;</span><span class="p">,</span>
+                         <span class="s1">&#39;mapreduce.job.queuename={}&#39;</span>
+                         <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">mapred_queue</span><span class="p">)])</span>
+
+                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mapred_queue_priority</span><span class="p">:</span>
+                    <span class="n">hive_conf_params</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span>
+                        <span class="p">[</span><span class="s1">&#39;-hiveconf&#39;</span><span class="p">,</span>
+                         <span class="s1">&#39;mapreduce.job.priority={}&#39;</span>
+                         <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">mapred_queue_priority</span><span class="p">)])</span>
+
+                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mapred_job_name</span><span class="p">:</span>
+                    <span class="n">hive_conf_params</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span>
+                        <span class="p">[</span><span class="s1">&#39;-hiveconf&#39;</span><span class="p">,</span>
+                         <span class="s1">&#39;mapred.job.name={}&#39;</span>
+                         <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">mapred_job_name</span><span class="p">)])</span>
+
+                <span class="n">hive_cmd</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">hive_conf_params</span><span class="p">)</span>
+                <span class="n">hive_cmd</span><span class="o">.</span><span class="n">extend</span><span class="p">([</span><span class="s1">&#39;-f&#39;</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">name</span><span class="p">])</span>
+
                 <span class="k">if</span> <span class="n">verbose</span><span class="p">:</span>
                     <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot; &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">hive_cmd</span><span class="p">))</span>
                 <span class="n">sp</span> <span class="o">=</span> <span class="n">subprocess</span><span class="o">.</span><span class="n">Popen</span><span class="p">(</span>
@@ -399,6 +475,8 @@
         <span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s1">&#39;sp&#39;</span><span class="p">):</span>
             <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">sp</span><span class="o">.</span><span class="n">poll</span><span class="p">()</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
                 <span class="k">print</span><span class="p">(</span><span class="s2">&quot;Killing the Hive job&quot;</span><span class="p">)</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">sp</span><span class="o">.</span><span class="n">terminate</span><span class="p">()</span>
+                <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">60</span><span class="p">)</span>
                 <span class="bp">self</span><span class="o">.</span><span class="n">sp</span><span class="o">.</span><span class="n">kill</span><span class="p">()</span></div>
 
 
@@ -700,11 +778,12 @@
                 <span class="n">cur</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">hql</span><span class="p">)</span>
                 <span class="n">schema</span> <span class="o">=</span> <span class="n">cur</span><span class="o">.</span><span class="n">description</span>
                 <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">csv_filepath</span><span class="p">,</span> <span class="s1">&#39;wb&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
-                    <span class="n">writer</span> <span class="o">=</span> <span class="n">csv</span><span class="o">.</span><span class="n">writer</span><span class="p">(</span><span class="n">f</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="n">delimiter</span><span class="p">,</span>
-                        <span class="n">lineterminator</span><span class="o">=</span><span class="n">lineterminator</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span>
+                    <span class="n">writer</span> <span class="o">=</span> <span class="n">csv</span><span class="o">.</span><span class="n">writer</span><span class="p">(</span><span class="n">f</span><span class="p">,</span>
+                                        <span class="n">delimiter</span><span class="o">=</span><span class="n">delimiter</span><span class="p">,</span>
+                                        <span class="n">lineterminator</span><span class="o">=</span><span class="n">lineterminator</span><span class="p">,</span>
+                                        <span class="n">encoding</span><span class="o">=</span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span>
                     <span class="k">if</span> <span class="n">output_header</span><span class="p">:</span>
-                        <span class="n">writer</span><span class="o">.</span><span class="n">writerow</span><span class="p">([</span><span class="n">c</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
-                            <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">cur</span><span class="o">.</span><span class="n">description</span><span class="p">])</span>
+                        <span class="n">writer</span><span class="o">.</span><span class="n">writerow</span><span class="p">([</span><span class="n">c</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">cur</span><span class="o">.</span><span class="n">description</span><span class="p">])</span>
                     <span class="n">i</span> <span class="o">=</span> <span class="mi">0</span>
                     <span class="k">while</span> <span class="bp">True</span><span class="p">:</span>
                         <span class="n">rows</span> <span class="o">=</span> <span class="p">[</span><span class="n">row</span> <span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">cur</span><span class="o">.</span><span class="n">fetchmany</span><span class="p">(</span><span class="n">fetch_size</span><span class="p">)</span> <span class="k">if</span> <span class="n">row</span><span class="p">]</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/hive_operator.html
----------------------------------------------------------------------
diff --git a/_modules/hive_operator.html b/_modules/hive_operator.html
index de5e44b..4d8c62c 100644
--- a/_modules/hive_operator.html
+++ b/_modules/hive_operator.html
@@ -177,6 +177,14 @@
 <span class="sd">    :param script_begin_tag: If defined, the operator will get rid of the</span>
 <span class="sd">        part of the script before the first occurrence of `script_begin_tag`</span>
 <span class="sd">    :type script_begin_tag: str</span>
+<span class="sd">    :param mapred_queue: queue used by the Hadoop CapacityScheduler</span>
+<span class="sd">    :type  mapred_queue: string</span>
+<span class="sd">    :param mapred_queue_priority: priority within CapacityScheduler queue.</span>
+<span class="sd">        Possible settings include: VERY_HIGH, HIGH, NORMAL, LOW, VERY_LOW</span>
+<span class="sd">    :type  mapred_queue_priority: string</span>
+<span class="sd">    :param mapred_job_name: This name will appear in the jobtracker.</span>
+<span class="sd">        This can make monitoring easier.</span>
+<span class="sd">    :type  mapred_job_name: string</span>
 <span class="sd">    &quot;&quot;&quot;</span>
 
     <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;hql&#39;</span><span class="p">,</span> <span class="s1">&#39;schema&#39;</span><span class="p">)</span>
@@ -191,6 +199,9 @@
             <span class="n">hiveconf_jinja_translate</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
             <span class="n">script_begin_tag</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
             <span class="n">run_as_owner</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
+            <span class="n">mapred_queue</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+            <span class="n">mapred_queue_priority</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+            <span class="n">mapred_job_name</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
             <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
 
         <span class="nb">super</span><span class="p">(</span><span class="n">HiveOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
@@ -203,8 +214,17 @@
         <span class="k">if</span> <span class="n">run_as_owner</span><span class="p">:</span>
             <span class="bp">self</span><span class="o">.</span><span class="n">run_as</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag</span><span class="o">.</span><span class="n">owner</span>
 
+        <span class="bp">self</span><span class="o">.</span><span class="n">mapred_queue</span> <span class="o">=</span> <span class="n">mapred_queue</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mapred_queue_priority</span> <span class="o">=</span> <span class="n">mapred_queue_priority</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mapred_job_name</span> <span class="o">=</span> <span class="n">mapred_job_name</span>
+
     <span class="k">def</span> <span class="nf">get_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
-        <span class="k">return</span> <span class="n">HiveCliHook</span><span class="p">(</span><span class="n">hive_cli_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">hive_cli_conn_id</span><span class="p">,</span> <span class="n">run_as</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">run_as</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">HiveCliHook</span><span class="p">(</span>
+                        <span class="n">hive_cli_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">hive_cli_conn_id</span><span class="p">,</span>
+                        <span class="n">run_as</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">run_as</span><span class="p">,</span>
+                        <span class="n">mapred_queue</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mapred_queue</span><span class="p">,</span>
+                        <span class="n">mapred_queue_priority</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mapred_queue_priority</span><span class="p">,</span>
+                        <span class="n">mapred_job_name</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mapred_job_name</span><span class="p">)</span>
 
     <span class="k">def</span> <span class="nf">prepare_template</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
         <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">hiveconf_jinja_translate</span><span class="p">:</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/hive_to_mysql.html
----------------------------------------------------------------------
diff --git a/_modules/hive_to_mysql.html b/_modules/hive_to_mysql.html
index 867023b..fd8b342 100644
--- a/_modules/hive_to_mysql.html
+++ b/_modules/hive_to_mysql.html
@@ -160,6 +160,7 @@
 
 <span class="kn">from</span> <span class="nn">tempfile</span> <span class="kn">import</span> <span class="n">NamedTemporaryFile</span>
 
+
 <div class="viewcode-block" id="HiveToMySqlTransfer"><a class="viewcode-back" href="../code.html#airflow.operators.HiveToMySqlTransfer">[docs]</a><span class="k">class</span> <span class="nc">HiveToMySqlTransfer</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
     <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Moves data from Hive to MySQL, note that for now the data is loaded</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/mysql_hook.html
----------------------------------------------------------------------
diff --git a/_modules/mysql_hook.html b/_modules/mysql_hook.html
index 941737f..a07aeb2 100644
--- a/_modules/mysql_hook.html
+++ b/_modules/mysql_hook.html
@@ -156,6 +156,7 @@
 
 <span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="kn">import</span> <span class="n">DbApiHook</span>
 
+
 <div class="viewcode-block" id="MySqlHook"><a class="viewcode-back" href="../code.html#airflow.hooks.MySqlHook">[docs]</a><span class="k">class</span> <span class="nc">MySqlHook</span><span class="p">(</span><span class="n">DbApiHook</span><span class="p">):</span>
     <span class="sd">&#39;&#39;&#39;</span>
 <span class="sd">    Interact with MySQL.</span>

[6/7] incubator-airflow-site git commit: New version of the docs

[ma...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/_modules/airflow/models.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/models.html b/_modules/airflow/models.html
index d4ed8a6..2f044a4 100644
--- a/_modules/airflow/models.html
+++ b/_modules/airflow/models.html
@@ -198,6 +198,7 @@
 <span class="kn">from</span> <span class="nn">airflow.executors</span> <span class="kn">import</span> <span class="n">DEFAULT_EXECUTOR</span><span class="p">,</span> <span class="n">LocalExecutor</span>
 <span class="kn">from</span> <span class="nn">airflow</span> <span class="kn">import</span> <span class="n">configuration</span>
 <span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span><span class="p">,</span> <span class="n">AirflowSkipException</span>
+<span class="kn">from</span> <span class="nn">airflow.dag.base_dag</span> <span class="kn">import</span> <span class="n">BaseDag</span><span class="p">,</span> <span class="n">BaseDagBag</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.dates</span> <span class="kn">import</span> <span class="n">cron_presets</span><span class="p">,</span> <span class="n">date_range</span> <span class="k">as</span> <span class="n">utils_date_range</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.db</span> <span class="kn">import</span> <span class="n">provide_session</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
@@ -205,6 +206,7 @@
 <span class="kn">from</span> <span class="nn">airflow.utils.helpers</span> <span class="kn">import</span> <span class="p">(</span>
     <span class="n">as_tuple</span><span class="p">,</span> <span class="n">is_container</span><span class="p">,</span> <span class="n">is_in</span><span class="p">,</span> <span class="n">validate_key</span><span class="p">,</span> <span class="n">pprinttable</span><span class="p">)</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.logging</span> <span class="kn">import</span> <span class="n">LoggingMixin</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.operator_resources</span> <span class="kn">import</span> <span class="n">Resources</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="kn">import</span> <span class="n">State</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.timeout</span> <span class="kn">import</span> <span class="n">timeout</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.trigger_rule</span> <span class="kn">import</span> <span class="n">TriggerRule</span>
@@ -267,7 +269,7 @@
             <span class="n">dr</span><span class="o">.</span><span class="n">start_date</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span>
 
 
-<div class="viewcode-block" id="DagBag"><a class="viewcode-back" href="../../code.html#airflow.models.DagBag">[docs]</a><span class="k">class</span> <span class="nc">DagBag</span><span class="p">(</span><span class="n">LoggingMixin</span><span class="p">):</span>
+<div class="viewcode-block" id="DagBag"><a class="viewcode-back" href="../../code.html#airflow.models.DagBag">[docs]</a><span class="k">class</span> <span class="nc">DagBag</span><span class="p">(</span><span class="n">BaseDagBag</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
     <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    A dagbag is a collection of dags, parsed out of a folder tree and has high</span>
 <span class="sd">    level configuration settings, like what database to use as a backend and</span>
@@ -278,7 +280,7 @@
 <span class="sd">    independent settings sets.</span>
 
 <span class="sd">    :param dag_folder: the folder to scan to find DAGs</span>
-<span class="sd">    :type dag_folder: str</span>
+<span class="sd">    :type dag_folder: unicode</span>
 <span class="sd">    :param executor: the executor to use when executing task instances</span>
 <span class="sd">        in this DagBag</span>
 <span class="sd">    :param include_examples: whether to include the examples that ship</span>
@@ -293,25 +295,23 @@
             <span class="bp">self</span><span class="p">,</span>
             <span class="n">dag_folder</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
             <span class="n">executor</span><span class="o">=</span><span class="n">DEFAULT_EXECUTOR</span><span class="p">,</span>
-            <span class="n">include_examples</span><span class="o">=</span><span class="n">configuration</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">&#39;core&#39;</span><span class="p">,</span> <span class="s1">&#39;LOAD_EXAMPLES&#39;</span><span class="p">),</span>
-            <span class="n">sync_to_db</span><span class="o">=</span><span class="bp">False</span><span class="p">):</span>
+            <span class="n">include_examples</span><span class="o">=</span><span class="n">configuration</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">&#39;core&#39;</span><span class="p">,</span> <span class="s1">&#39;LOAD_EXAMPLES&#39;</span><span class="p">)):</span>
 
         <span class="n">dag_folder</span> <span class="o">=</span> <span class="n">dag_folder</span> <span class="ow">or</span> <span class="n">DAGS_FOLDER</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Filling up the DagBag from {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">dag_folder</span><span class="p">))</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">dag_folder</span> <span class="o">=</span> <span class="n">dag_folder</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">dags</span> <span class="o">=</span> <span class="p">{}</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">sync_to_db</span> <span class="o">=</span> <span class="n">sync_to_db</span>
+        <span class="c1"># the file&#39;s last modified timestamp when we last read it</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">file_last_changed</span> <span class="o">=</span> <span class="p">{}</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">executor</span> <span class="o">=</span> <span class="n">executor</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">import_errors</span> <span class="o">=</span> <span class="p">{}</span>
+
         <span class="k">if</span> <span class="n">include_examples</span><span class="p">:</span>
             <span class="n">example_dag_folder</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
                 <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">dirname</span><span class="p">(</span><span class="n">__file__</span><span class="p">),</span>
                 <span class="s1">&#39;example_dags&#39;</span><span class="p">)</span>
             <span class="bp">self</span><span class="o">.</span><span class="n">collect_dags</span><span class="p">(</span><span class="n">example_dag_folder</span><span class="p">)</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">collect_dags</span><span class="p">(</span><span class="n">dag_folder</span><span class="p">)</span>
-        <span class="k">if</span> <span class="n">sync_to_db</span><span class="p">:</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">deactivate_inactive_dags</span><span class="p">()</span>
 
 <div class="viewcode-block" id="DagBag.size"><a class="viewcode-back" href="../../code.html#airflow.models.DagBag.size">[docs]</a>    <span class="k">def</span> <span class="nf">size</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
@@ -330,7 +330,7 @@
             <span class="k">if</span> <span class="n">dag</span><span class="o">.</span><span class="n">is_subdag</span><span class="p">:</span>
                 <span class="n">root_dag_id</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">parent_dag</span><span class="o">.</span><span class="n">dag_id</span>
 
-        <span class="c1"># If the root_dag_id is absent or expired</span>
+        <span class="c1"># If the dag corresponding to root_dag_id is absent or expired</span>
         <span class="n">orm_dag</span> <span class="o">=</span> <span class="n">DagModel</span><span class="o">.</span><span class="n">get_current</span><span class="p">(</span><span class="n">root_dag_id</span><span class="p">)</span>
         <span class="k">if</span> <span class="n">orm_dag</span> <span class="ow">and</span> <span class="p">(</span>
                 <span class="n">root_dag_id</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">dags</span> <span class="ow">or</span>
@@ -339,10 +339,11 @@
                     <span class="n">dag</span><span class="o">.</span><span class="n">last_loaded</span> <span class="o">&lt;</span> <span class="n">orm_dag</span><span class="o">.</span><span class="n">last_expired</span>
                 <span class="p">)</span>
         <span class="p">):</span>
-            <span class="c1"># Reprocessing source file</span>
+            <span class="c1"># Reprocess source file</span>
             <span class="n">found_dags</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">process_file</span><span class="p">(</span>
                 <span class="n">filepath</span><span class="o">=</span><span class="n">orm_dag</span><span class="o">.</span><span class="n">fileloc</span><span class="p">,</span> <span class="n">only_if_updated</span><span class="o">=</span><span class="bp">False</span><span class="p">)</span>
 
+            <span class="c1"># If the source file no longer exports `dag_id`, delete it from self.dags</span>
             <span class="k">if</span> <span class="n">found_dags</span> <span class="ow">and</span> <span class="n">dag_id</span> <span class="ow">in</span> <span class="p">[</span><span class="n">dag</span><span class="o">.</span><span class="n">dag_id</span> <span class="k">for</span> <span class="n">dag</span> <span class="ow">in</span> <span class="n">found_dags</span><span class="p">]:</span>
                 <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">dags</span><span class="p">[</span><span class="n">dag_id</span><span class="p">]</span>
             <span class="k">elif</span> <span class="n">dag_id</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">dags</span><span class="p">:</span>
@@ -363,10 +364,10 @@
         <span class="k">try</span><span class="p">:</span>
             <span class="c1"># This failed before in what may have been a git sync</span>
             <span class="c1"># race condition</span>
-            <span class="n">dttm</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">getmtime</span><span class="p">(</span><span class="n">filepath</span><span class="p">))</span>
+            <span class="n">file_last_changed_on_disk</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">getmtime</span><span class="p">(</span><span class="n">filepath</span><span class="p">))</span>
             <span class="k">if</span> <span class="n">only_if_updated</span> \
                     <span class="ow">and</span> <span class="n">filepath</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">file_last_changed</span> \
-                    <span class="ow">and</span> <span class="n">dttm</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">file_last_changed</span><span class="p">[</span><span class="n">filepath</span><span class="p">]:</span>
+                    <span class="ow">and</span> <span class="n">file_last_changed_on_disk</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">file_last_changed</span><span class="p">[</span><span class="n">filepath</span><span class="p">]:</span>
                 <span class="k">return</span> <span class="n">found_dags</span>
 
         <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
@@ -395,7 +396,7 @@
                 <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
                     <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="s2">&quot;Failed to import: &quot;</span> <span class="o">+</span> <span class="n">filepath</span><span class="p">)</span>
                     <span class="bp">self</span><span class="o">.</span><span class="n">import_errors</span><span class="p">[</span><span class="n">filepath</span><span class="p">]</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">)</span>
-                    <span class="bp">self</span><span class="o">.</span><span class="n">file_last_changed</span><span class="p">[</span><span class="n">filepath</span><span class="p">]</span> <span class="o">=</span> <span class="n">dttm</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">file_last_changed</span><span class="p">[</span><span class="n">filepath</span><span class="p">]</span> <span class="o">=</span> <span class="n">file_last_changed_on_disk</span>
 
         <span class="k">else</span><span class="p">:</span>
             <span class="n">zip_file</span> <span class="o">=</span> <span class="n">zipfile</span><span class="o">.</span><span class="n">ZipFile</span><span class="p">(</span><span class="n">filepath</span><span class="p">)</span>
@@ -426,7 +427,7 @@
                     <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
                         <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="s2">&quot;Failed to import: &quot;</span> <span class="o">+</span> <span class="n">filepath</span><span class="p">)</span>
                         <span class="bp">self</span><span class="o">.</span><span class="n">import_errors</span><span class="p">[</span><span class="n">filepath</span><span class="p">]</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">)</span>
-                        <span class="bp">self</span><span class="o">.</span><span class="n">file_last_changed</span><span class="p">[</span><span class="n">filepath</span><span class="p">]</span> <span class="o">=</span> <span class="n">dttm</span>
+                        <span class="bp">self</span><span class="o">.</span><span class="n">file_last_changed</span><span class="p">[</span><span class="n">filepath</span><span class="p">]</span> <span class="o">=</span> <span class="n">file_last_changed_on_disk</span>
 
         <span class="k">for</span> <span class="n">m</span> <span class="ow">in</span> <span class="n">mods</span><span class="p">:</span>
             <span class="k">for</span> <span class="n">dag</span> <span class="ow">in</span> <span class="nb">list</span><span class="p">(</span><span class="n">m</span><span class="o">.</span><span class="n">__dict__</span><span class="o">.</span><span class="n">values</span><span class="p">()):</span>
@@ -439,11 +440,11 @@
                     <span class="n">found_dags</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">dag</span><span class="p">)</span>
                     <span class="n">found_dags</span> <span class="o">+=</span> <span class="n">dag</span><span class="o">.</span><span class="n">subdags</span>
 
-        <span class="bp">self</span><span class="o">.</span><span class="n">file_last_changed</span><span class="p">[</span><span class="n">filepath</span><span class="p">]</span> <span class="o">=</span> <span class="n">dttm</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">file_last_changed</span><span class="p">[</span><span class="n">filepath</span><span class="p">]</span> <span class="o">=</span> <span class="n">file_last_changed_on_disk</span>
         <span class="k">return</span> <span class="n">found_dags</span></div>
 
     <span class="nd">@provide_session</span>
-<div class="viewcode-block" id="DagBag.kill_zombies"><a class="viewcode-back" href="../../code.html#airflow.models.DagBag.kill_zombies">[docs]</a>    <span class="k">def</span> <span class="nf">kill_zombies</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="p">):</span>
+<div class="viewcode-block" id="DagBag.kill_zombies"><a class="viewcode-back" href="../../code.html#airflow.models.DagBag.kill_zombies">[docs]</a>    <span class="k">def</span> <span class="nf">kill_zombies</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Fails tasks that haven&#39;t had a heartbeat in too long</span>
 <span class="sd">        &quot;&quot;&quot;</span>
@@ -477,6 +478,7 @@
                     <span class="n">ti</span><span class="o">.</span><span class="n">handle_failure</span><span class="p">(</span><span class="s2">&quot;{} killed as zombie&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">ti</span><span class="p">))</span>
                     <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
                         <span class="s1">&#39;Marked zombie job {} as failed&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">ti</span><span class="p">))</span>
+                    <span class="n">Stats</span><span class="o">.</span><span class="n">incr</span><span class="p">(</span><span class="s1">&#39;zombies_killed&#39;</span><span class="p">)</span>
         <span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span></div>
 
 <div class="viewcode-block" id="DagBag.bag_dag"><a class="viewcode-back" href="../../code.html#airflow.models.DagBag.bag_dag">[docs]</a>    <span class="k">def</span> <span class="nf">bag_dag</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dag</span><span class="p">,</span> <span class="n">parent_dag</span><span class="p">,</span> <span class="n">root_dag</span><span class="p">):</span>
@@ -490,20 +492,6 @@
         <span class="k">for</span> <span class="n">task</span> <span class="ow">in</span> <span class="n">dag</span><span class="o">.</span><span class="n">tasks</span><span class="p">:</span>
             <span class="n">settings</span><span class="o">.</span><span class="n">policy</span><span class="p">(</span><span class="n">task</span><span class="p">)</span>
 
-        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">sync_to_db</span><span class="p">:</span>
-            <span class="n">session</span> <span class="o">=</span> <span class="n">settings</span><span class="o">.</span><span class="n">Session</span><span class="p">()</span>
-            <span class="n">orm_dag</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span>
-                <span class="n">DagModel</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DagModel</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="n">dag</span><span class="o">.</span><span class="n">dag_id</span><span class="p">)</span><span class="o">.</span><span class="n">first</span><span class="p">()</span>
-            <span class="k">if</span> <span class="ow">not</span> <span class="n">orm_dag</span><span class="p">:</span>
-                <span class="n">orm_dag</span> <span class="o">=</span> <span class="n">DagModel</span><span class="p">(</span><span class="n">dag_id</span><span class="o">=</span><span class="n">dag</span><span class="o">.</span><span class="n">dag_id</span><span class="p">)</span>
-            <span class="n">orm_dag</span><span class="o">.</span><span class="n">fileloc</span> <span class="o">=</span> <span class="n">root_dag</span><span class="o">.</span><span class="n">full_filepath</span>
-            <span class="n">orm_dag</span><span class="o">.</span><span class="n">is_subdag</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">is_subdag</span>
-            <span class="n">orm_dag</span><span class="o">.</span><span class="n">owners</span> <span class="o">=</span> <span class="n">root_dag</span><span class="o">.</span><span class="n">owner</span>
-            <span class="n">orm_dag</span><span class="o">.</span><span class="n">is_active</span> <span class="o">=</span> <span class="bp">True</span>
-            <span class="n">session</span><span class="o">.</span><span class="n">merge</span><span class="p">(</span><span class="n">orm_dag</span><span class="p">)</span>
-            <span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
-            <span class="n">session</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
-
         <span class="k">for</span> <span class="n">subdag</span> <span class="ow">in</span> <span class="n">dag</span><span class="o">.</span><span class="n">subdags</span><span class="p">:</span>
             <span class="n">subdag</span><span class="o">.</span><span class="n">full_filepath</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">full_filepath</span>
             <span class="n">subdag</span><span class="o">.</span><span class="n">parent_dag</span> <span class="o">=</span> <span class="n">dag</span>
@@ -692,7 +680,8 @@
         <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_password</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_encrypted</span><span class="p">:</span>
             <span class="k">if</span> <span class="ow">not</span> <span class="n">ENCRYPTION_ON</span><span class="p">:</span>
                 <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
-                    <span class="s2">&quot;Can&#39;t decrypt, configuration is missing&quot;</span><span class="p">)</span>
+                    <span class="s2">&quot;Can&#39;t decrypt encrypted password for login={}, </span><span class="se">\</span>
+<span class="s2">                    FERNET_KEY configuration is missing&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">login</span><span class="p">))</span>
             <span class="k">return</span> <span class="n">FERNET</span><span class="o">.</span><span class="n">decrypt</span><span class="p">(</span><span class="nb">bytes</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_password</span><span class="p">,</span> <span class="s1">&#39;utf-8&#39;</span><span class="p">))</span><span class="o">.</span><span class="n">decode</span><span class="p">()</span>
         <span class="k">else</span><span class="p">:</span>
             <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_password</span>
@@ -715,7 +704,8 @@
         <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_extra</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_extra_encrypted</span><span class="p">:</span>
             <span class="k">if</span> <span class="ow">not</span> <span class="n">ENCRYPTION_ON</span><span class="p">:</span>
                 <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
-                    <span class="s2">&quot;Can&#39;t decrypt `extra`, configuration is missing&quot;</span><span class="p">)</span>
+                    <span class="s2">&quot;Can&#39;t decrypt `extra` params for login={},</span><span class="se">\</span>
+<span class="s2">                    FERNET_KEY configuration is missing&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">login</span><span class="p">))</span>
             <span class="k">return</span> <span class="n">FERNET</span><span class="o">.</span><span class="n">decrypt</span><span class="p">(</span><span class="nb">bytes</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_extra</span><span class="p">,</span> <span class="s1">&#39;utf-8&#39;</span><span class="p">))</span><span class="o">.</span><span class="n">decode</span><span class="p">()</span>
         <span class="k">else</span><span class="p">:</span>
             <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_extra</span>
@@ -891,8 +881,78 @@
 <span class="sd">        the orchestrator.</span>
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="n">dag</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="o">.</span><span class="n">dag</span>
-        <span class="n">iso</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span>
-        <span class="n">cmd</span> <span class="o">=</span> <span class="s2">&quot;airflow run {self.dag_id} {self.task_id} {iso} &quot;</span>
+
+        <span class="n">should_pass_filepath</span> <span class="o">=</span> <span class="ow">not</span> <span class="n">pickle_id</span> <span class="ow">and</span> <span class="n">dag</span>
+        <span class="k">if</span> <span class="n">should_pass_filepath</span> <span class="ow">and</span> <span class="n">dag</span><span class="o">.</span><span class="n">full_filepath</span> <span class="o">!=</span> <span class="n">dag</span><span class="o">.</span><span class="n">filepath</span><span class="p">:</span>
+            <span class="n">path</span> <span class="o">=</span> <span class="s2">&quot;DAGS_FOLDER/{}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">dag</span><span class="o">.</span><span class="n">filepath</span><span class="p">)</span>
+        <span class="k">elif</span> <span class="n">should_pass_filepath</span> <span class="ow">and</span> <span class="n">dag</span><span class="o">.</span><span class="n">full_filepath</span><span class="p">:</span>
+            <span class="n">path</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">full_filepath</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">path</span> <span class="o">=</span> <span class="bp">None</span>
+
+        <span class="k">return</span> <span class="n">TaskInstance</span><span class="o">.</span><span class="n">generate_command</span><span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="p">,</span>
+            <span class="n">mark_success</span><span class="o">=</span><span class="n">mark_success</span><span class="p">,</span>
+            <span class="n">ignore_dependencies</span><span class="o">=</span><span class="n">ignore_dependencies</span><span class="p">,</span>
+            <span class="n">ignore_depends_on_past</span><span class="o">=</span><span class="n">ignore_depends_on_past</span><span class="p">,</span>
+            <span class="n">force</span><span class="o">=</span><span class="n">force</span><span class="p">,</span>
+            <span class="n">local</span><span class="o">=</span><span class="n">local</span><span class="p">,</span>
+            <span class="n">pickle_id</span><span class="o">=</span><span class="n">pickle_id</span><span class="p">,</span>
+            <span class="n">file_path</span><span class="o">=</span><span class="n">path</span><span class="p">,</span>
+            <span class="n">raw</span><span class="o">=</span><span class="n">raw</span><span class="p">,</span>
+            <span class="n">job_id</span><span class="o">=</span><span class="n">job_id</span><span class="p">,</span>
+            <span class="n">pool</span><span class="o">=</span><span class="n">pool</span><span class="p">)</span></div>
+
+    <span class="nd">@staticmethod</span>
+<div class="viewcode-block" id="TaskInstance.generate_command"><a class="viewcode-back" href="../../code.html#airflow.models.TaskInstance.generate_command">[docs]</a>    <span class="k">def</span> <span class="nf">generate_command</span><span class="p">(</span><span class="n">dag_id</span><span class="p">,</span>
+                         <span class="n">task_id</span><span class="p">,</span>
+                         <span class="n">execution_date</span><span class="p">,</span>
+                         <span class="n">mark_success</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
+                         <span class="n">ignore_dependencies</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
+                         <span class="n">ignore_depends_on_past</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
+                         <span class="n">force</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
+                         <span class="n">local</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
+                         <span class="n">pickle_id</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+                         <span class="n">file_path</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+                         <span class="n">raw</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
+                         <span class="n">job_id</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+                         <span class="n">pool</span><span class="o">=</span><span class="bp">None</span>
+                         <span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Generates the shell command required to execute this task instance.</span>
+
+<span class="sd">        :param dag_id: DAG ID</span>
+<span class="sd">        :type dag_id: unicode</span>
+<span class="sd">        :param task_id: Task ID</span>
+<span class="sd">        :type task_id: unicode</span>
+<span class="sd">        :param execution_date: Execution date for the task</span>
+<span class="sd">        :type execution_date: datetime</span>
+<span class="sd">        :param mark_success: Whether to mark the task as successful</span>
+<span class="sd">        :type mark_success: bool</span>
+<span class="sd">        :param ignore_dependencies: Whether to ignore the dependencies and run</span>
+<span class="sd">        anyway</span>
+<span class="sd">        :type ignore_dependencies: bool</span>
+<span class="sd">        :param ignore_depends_on_past: Whether to ignore the depends on past</span>
+<span class="sd">        setting and run anyway</span>
+<span class="sd">        :type ignore_depends_on_past: bool</span>
+<span class="sd">        :param force: Whether to force running - see TaskInstance.run()</span>
+<span class="sd">        :type force: bool</span>
+<span class="sd">        :param local: Whether to run the task locally</span>
+<span class="sd">        :type local: bool</span>
+<span class="sd">        :param pickle_id: If the DAG was serialized to the DB, the ID</span>
+<span class="sd">        associated with the pickled DAG</span>
+<span class="sd">        :type pickle_id: unicode</span>
+<span class="sd">        :param file_path: path to the file containing the DAG definition</span>
+<span class="sd">        :param raw: raw mode (needs more details)</span>
+<span class="sd">        :param job_id: job ID (needs more details)</span>
+<span class="sd">        :param pool: the Airflow pool that the task should run in</span>
+<span class="sd">        :type pool: unicode</span>
+<span class="sd">        :return: shell command that can be used to run the task instance</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">iso</span> <span class="o">=</span> <span class="n">execution_date</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span>
+        <span class="n">cmd</span> <span class="o">=</span> <span class="s2">&quot;airflow run {dag_id} {task_id} {iso} &quot;</span>
         <span class="n">cmd</span> <span class="o">+=</span> <span class="s2">&quot;--mark_success &quot;</span> <span class="k">if</span> <span class="n">mark_success</span> <span class="k">else</span> <span class="s2">&quot;&quot;</span>
         <span class="n">cmd</span> <span class="o">+=</span> <span class="s2">&quot;--pickle {pickle_id} &quot;</span> <span class="k">if</span> <span class="n">pickle_id</span> <span class="k">else</span> <span class="s2">&quot;&quot;</span>
         <span class="n">cmd</span> <span class="o">+=</span> <span class="s2">&quot;--job_id {job_id} &quot;</span> <span class="k">if</span> <span class="n">job_id</span> <span class="k">else</span> <span class="s2">&quot;&quot;</span>
@@ -902,11 +962,7 @@
         <span class="n">cmd</span> <span class="o">+=</span> <span class="s2">&quot;--local &quot;</span> <span class="k">if</span> <span class="n">local</span> <span class="k">else</span> <span class="s2">&quot;&quot;</span>
         <span class="n">cmd</span> <span class="o">+=</span> <span class="s2">&quot;--pool {pool} &quot;</span> <span class="k">if</span> <span class="n">pool</span> <span class="k">else</span> <span class="s2">&quot;&quot;</span>
         <span class="n">cmd</span> <span class="o">+=</span> <span class="s2">&quot;--raw &quot;</span> <span class="k">if</span> <span class="n">raw</span> <span class="k">else</span> <span class="s2">&quot;&quot;</span>
-        <span class="k">if</span> <span class="ow">not</span> <span class="n">pickle_id</span> <span class="ow">and</span> <span class="n">dag</span><span class="p">:</span>
-            <span class="k">if</span> <span class="n">dag</span><span class="o">.</span><span class="n">full_filepath</span> <span class="o">!=</span> <span class="n">dag</span><span class="o">.</span><span class="n">filepath</span><span class="p">:</span>
-                <span class="n">cmd</span> <span class="o">+=</span> <span class="s2">&quot;-sd DAGS_FOLDER/{dag.filepath} &quot;</span>
-            <span class="k">elif</span> <span class="n">dag</span><span class="o">.</span><span class="n">full_filepath</span><span class="p">:</span>
-                <span class="n">cmd</span> <span class="o">+=</span> <span class="s2">&quot;-sd {dag.full_filepath}&quot;</span>
+        <span class="n">cmd</span> <span class="o">+=</span> <span class="s2">&quot;-sd {file_path}&quot;</span> <span class="k">if</span> <span class="n">file_path</span> <span class="k">else</span> <span class="s2">&quot;&quot;</span>
         <span class="k">return</span> <span class="n">cmd</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">())</span></div>
 
     <span class="nd">@property</span>
@@ -1284,13 +1340,25 @@
             <span class="s2">&quot;{ti.execution_date} [{ti.state}]&gt;&quot;</span>
         <span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">ti</span><span class="o">=</span><span class="bp">self</span><span class="p">)</span>
 
+<div class="viewcode-block" id="TaskInstance.next_retry_datetime"><a class="viewcode-back" href="../../code.html#airflow.models.TaskInstance.next_retry_datetime">[docs]</a>    <span class="k">def</span> <span class="nf">next_retry_datetime</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Get datetime of the next retry if the task instance fails. For exponential</span>
+<span class="sd">        backoff, retry_delay is used as base and will be converted to seconds.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">delay</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="o">.</span><span class="n">retry_delay</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="o">.</span><span class="n">retry_exponential_backoff</span><span class="p">:</span>
+            <span class="n">delay_backoff_in_seconds</span> <span class="o">=</span> <span class="n">delay</span><span class="o">.</span><span class="n">total_seconds</span><span class="p">()</span> <span class="o">**</span> <span class="bp">self</span><span class="o">.</span><span class="n">try_number</span>
+            <span class="n">delay</span> <span class="o">=</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">seconds</span><span class="o">=</span><span class="n">delay_backoff_in_seconds</span><span class="p">)</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="o">.</span><span class="n">max_retry_delay</span><span class="p">:</span>
+                <span class="n">delay</span> <span class="o">=</span> <span class="nb">min</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="o">.</span><span class="n">max_retry_delay</span><span class="p">,</span> <span class="n">delay</span><span class="p">)</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">end_date</span> <span class="o">+</span> <span class="n">delay</span></div>
+
 <div class="viewcode-block" id="TaskInstance.ready_for_retry"><a class="viewcode-back" href="../../code.html#airflow.models.TaskInstance.ready_for_retry">[docs]</a>    <span class="k">def</span> <span class="nf">ready_for_retry</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Checks on whether the task instance is in the right state and timeframe</span>
 <span class="sd">        to be retried.</span>
 <span class="sd">        &quot;&quot;&quot;</span>
-        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">State</span><span class="o">.</span><span class="n">UP_FOR_RETRY</span> <span class="ow">and</span> \
-            <span class="bp">self</span><span class="o">.</span><span class="n">end_date</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="o">.</span><span class="n">retry_delay</span> <span class="o">&lt;</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span></div>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">State</span><span class="o">.</span><span class="n">UP_FOR_RETRY</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">next_retry_datetime</span><span class="p">()</span> <span class="o">&lt;</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span></div>
 
     <span class="nd">@provide_session</span>
 <div class="viewcode-block" id="TaskInstance.pool_full"><a class="viewcode-back" href="../../code.html#airflow.models.TaskInstance.pool_full">[docs]</a>    <span class="k">def</span> <span class="nf">pool_full</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="p">):</span>
@@ -1308,9 +1376,7 @@
             <span class="o">.</span><span class="n">first</span><span class="p">()</span>
         <span class="p">)</span>
         <span class="k">if</span> <span class="ow">not</span> <span class="n">pool</span><span class="p">:</span>
-            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
-                <span class="s2">&quot;Task specified a pool ({}) but the pool &quot;</span>
-                <span class="s2">&quot;doesn&#39;t exist!&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="o">.</span><span class="n">pool</span><span class="p">))</span>
+            <span class="k">return</span> <span class="bp">False</span>
         <span class="n">open_slots</span> <span class="o">=</span> <span class="n">pool</span><span class="o">.</span><span class="n">open_slots</span><span class="p">(</span><span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">)</span>
 
         <span class="k">return</span> <span class="n">open_slots</span> <span class="o">&lt;=</span> <span class="mi">0</span></div>
@@ -1378,7 +1444,7 @@
             <span class="c1"># todo: move this to the scheduler</span>
                 <span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">State</span><span class="o">.</span><span class="n">UP_FOR_RETRY</span> <span class="ow">and</span>
                 <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">ready_for_retry</span><span class="p">()):</span>
-            <span class="n">next_run</span> <span class="o">=</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">end_date</span> <span class="o">+</span> <span class="n">task</span><span class="o">.</span><span class="n">retry_delay</span><span class="p">)</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span>
+            <span class="n">next_run</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">next_retry_datetime</span><span class="p">()</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span>
             <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
                 <span class="s2">&quot;Not ready for retry yet. &quot;</span> <span class="o">+</span>
                 <span class="s2">&quot;Next run after {0}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">next_run</span><span class="p">)</span>
@@ -1510,6 +1576,9 @@
         <span class="k">if</span> <span class="ow">not</span> <span class="n">test_mode</span><span class="p">:</span>
             <span class="n">session</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">Log</span><span class="p">(</span><span class="n">State</span><span class="o">.</span><span class="n">FAILED</span><span class="p">,</span> <span class="bp">self</span><span class="p">))</span>
 
+        <span class="c1"># Log failure duration</span>
+        <span class="n">session</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">TaskFail</span><span class="p">(</span><span class="n">task</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">start_date</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">end_date</span><span class="p">))</span>
+
         <span class="c1"># Let&#39;s go deeper</span>
         <span class="k">try</span><span class="p">:</span>
             <span class="k">if</span> <span class="n">task</span><span class="o">.</span><span class="n">retries</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">try_number</span> <span class="o">%</span> <span class="p">(</span><span class="n">task</span><span class="o">.</span><span class="n">retries</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
@@ -1592,17 +1661,25 @@
 <span class="sd">            {var.variable_name}.</span>
 <span class="sd">            &quot;&quot;&quot;</span>
             <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
-                <span class="k">pass</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">var</span> <span class="o">=</span> <span class="bp">None</span>
 
             <span class="k">def</span> <span class="nf">__getattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">item</span><span class="p">):</span>
-                <span class="k">return</span> <span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">var</span> <span class="o">=</span> <span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
+                <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">var</span>
+
+            <span class="k">def</span> <span class="nf">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+                <span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">var</span><span class="p">)</span>
 
         <span class="k">class</span> <span class="nc">VariableJsonAccessor</span><span class="p">:</span>
             <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
-                <span class="k">pass</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">var</span> <span class="o">=</span> <span class="bp">None</span>
 
             <span class="k">def</span> <span class="nf">__getattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">item</span><span class="p">):</span>
-                <span class="k">return</span> <span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="n">deserialize_json</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">var</span> <span class="o">=</span>  <span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="n">deserialize_json</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span>
+                <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">var</span>
+
+            <span class="k">def</span> <span class="nf">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+                <span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">var</span><span class="p">)</span>
 
         <span class="k">return</span> <span class="p">{</span>
             <span class="s1">&#39;dag&#39;</span><span class="p">:</span> <span class="n">task</span><span class="o">.</span><span class="n">dag</span><span class="p">,</span>
@@ -1755,6 +1832,29 @@
             <span class="k">return</span> <span class="n">pull_fn</span><span class="p">(</span><span class="n">task_id</span><span class="o">=</span><span class="n">task_ids</span><span class="p">)</span></div></div>
 
 
+<span class="k">class</span> <span class="nc">TaskFail</span><span class="p">(</span><span class="n">Base</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    TaskFail tracks the failed run durations of each task instance.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="n">__tablename__</span> <span class="o">=</span> <span class="s2">&quot;task_fail&quot;</span>
+
+    <span class="n">task_id</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="n">ID_LEN</span><span class="p">),</span> <span class="n">primary_key</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span>
+    <span class="n">dag_id</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="n">ID_LEN</span><span class="p">),</span> <span class="n">primary_key</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span>
+    <span class="n">execution_date</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">DateTime</span><span class="p">,</span> <span class="n">primary_key</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span>
+    <span class="n">start_date</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">DateTime</span><span class="p">)</span>
+    <span class="n">end_date</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">DateTime</span><span class="p">)</span>
+    <span class="n">duration</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Float</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">task</span><span class="p">,</span> <span class="n">execution_date</span><span class="p">,</span> <span class="n">start_date</span><span class="p">,</span> <span class="n">end_date</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">=</span> <span class="n">task</span><span class="o">.</span><span class="n">dag_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">task_id</span> <span class="o">=</span> <span class="n">task</span><span class="o">.</span><span class="n">task_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">=</span> <span class="n">execution_date</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">start_date</span> <span class="o">=</span> <span class="n">start_date</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">end_date</span> <span class="o">=</span> <span class="n">end_date</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">duration</span> <span class="o">=</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">end_date</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">start_date</span><span class="p">)</span><span class="o">.</span><span class="n">total_seconds</span><span class="p">()</span>
+
+
 <span class="k">class</span> <span class="nc">Log</span><span class="p">(</span><span class="n">Base</span><span class="p">):</span>
     <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Used to actively log events to the database</span>
@@ -1831,6 +1931,12 @@
 <span class="sd">    :type retries: int</span>
 <span class="sd">    :param retry_delay: delay between retries</span>
 <span class="sd">    :type retry_delay: timedelta</span>
+<span class="sd">    :param retry_exponential_backoff: allow progressive longer waits between</span>
+<span class="sd">        retries by using exponential backoff algorithm on retry delay (delay</span>
+<span class="sd">        will be converted into seconds)</span>
+<span class="sd">    :type retry_exponential_backoff: bool</span>
+<span class="sd">    :param max_retry_delay: maximum delay interval between retries</span>
+<span class="sd">    :type max_retry_delay: timedelta</span>
 <span class="sd">    :param start_date: The ``start_date`` for the task, determines</span>
 <span class="sd">        the ``execution_date`` for the first task instance. The best practice</span>
 <span class="sd">        is to have the start_date rounded</span>
@@ -1908,6 +2014,9 @@
 <span class="sd">        using the constants defined in the static class</span>
 <span class="sd">        ``airflow.utils.TriggerRule``</span>
 <span class="sd">    :type trigger_rule: str</span>
+<span class="sd">    :param resources: A map of resource parameter names (the argument names of the</span>
+<span class="sd">        Resources constructor) to their values.</span>
+<span class="sd">    :type resources: dict</span>
 <span class="sd">    &quot;&quot;&quot;</span>
 
     <span class="c1"># For derived classes to define which fields will get jinjaified</span>
@@ -1928,6 +2037,8 @@
             <span class="n">email_on_failure</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span>
             <span class="n">retries</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
             <span class="n">retry_delay</span><span class="o">=</span><span class="n">timedelta</span><span class="p">(</span><span class="n">seconds</span><span class="o">=</span><span class="mi">300</span><span class="p">),</span>
+            <span class="n">retry_exponential_backoff</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
+            <span class="n">max_retry_delay</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
             <span class="n">start_date</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
             <span class="n">end_date</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
             <span class="n">schedule_interval</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>  <span class="c1"># not hooked as of now</span>
@@ -1946,6 +2057,7 @@
             <span class="n">on_success_callback</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
             <span class="n">on_retry_callback</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
             <span class="n">trigger_rule</span><span class="o">=</span><span class="n">TriggerRule</span><span class="o">.</span><span class="n">ALL_SUCCESS</span><span class="p">,</span>
+            <span class="n">resources</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
             <span class="o">*</span><span class="n">args</span><span class="p">,</span>
             <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
 
@@ -2003,9 +2115,12 @@
         <span class="k">else</span><span class="p">:</span>
             <span class="n">logging</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;retry_delay isn&#39;t timedelta object, assuming secs&quot;</span><span class="p">)</span>
             <span class="bp">self</span><span class="o">.</span><span class="n">retry_delay</span> <span class="o">=</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">seconds</span><span class="o">=</span><span class="n">retry_delay</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">retry_exponential_backoff</span> <span class="o">=</span> <span class="n">retry_exponential_backoff</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">max_retry_delay</span> <span class="o">=</span> <span class="n">max_retry_delay</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">params</span> <span class="o">=</span> <span class="n">params</span> <span class="ow">or</span> <span class="p">{}</span>  <span class="c1"># Available in templates!</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">adhoc</span> <span class="o">=</span> <span class="n">adhoc</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">priority_weight</span> <span class="o">=</span> <span class="n">priority_weight</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">resources</span> <span class="o">=</span> <span class="n">Resources</span><span class="p">(</span><span class="o">**</span><span class="p">(</span><span class="n">resources</span> <span class="ow">or</span> <span class="p">{}))</span>
 
         <span class="c1"># Private attributes</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">_upstream_task_ids</span> <span class="o">=</span> <span class="p">[]</span>
@@ -2023,6 +2138,8 @@
             <span class="s1">&#39;email&#39;</span><span class="p">,</span>
             <span class="s1">&#39;email_on_retry&#39;</span><span class="p">,</span>
             <span class="s1">&#39;retry_delay&#39;</span><span class="p">,</span>
+            <span class="s1">&#39;retry_exponential_backoff&#39;</span><span class="p">,</span>
+            <span class="s1">&#39;max_retry_delay&#39;</span><span class="p">,</span>
             <span class="s1">&#39;start_date&#39;</span><span class="p">,</span>
             <span class="s1">&#39;schedule_interval&#39;</span><span class="p">,</span>
             <span class="s1">&#39;depends_on_past&#39;</span><span class="p">,</span>
@@ -2042,7 +2159,7 @@
             <span class="nb">all</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">__dict__</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">c</span><span class="p">,</span> <span class="bp">None</span><span class="p">)</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">__dict__</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">c</span><span class="p">,</span> <span class="bp">None</span><span class="p">)</span>
                 <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_comps</span><span class="p">))</span>
 
-    <span class="k">def</span> <span class="nf">__neq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
+    <span class="k">def</span> <span class="nf">__ne__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
         <span class="k">return</span> <span class="ow">not</span> <span class="bp">self</span> <span class="o">==</span> <span class="n">other</span>
 
     <span class="k">def</span> <span class="nf">__lt__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
@@ -2541,8 +2658,8 @@
     <span class="n">last_scheduler_run</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">DateTime</span><span class="p">)</span>
     <span class="c1"># Last time this DAG was pickled</span>
     <span class="n">last_pickled</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">DateTime</span><span class="p">)</span>
-    <span class="c1"># When the DAG received a refreshed signal last, used to know when</span>
-    <span class="c1"># we need to force refresh</span>
+    <span class="c1"># Time when the DAG last received a refresh signal</span>
+    <span class="c1"># (e.g. the DAG&#39;s &quot;refresh&quot; button was clicked in the web UI)</span>
     <span class="n">last_expired</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">DateTime</span><span class="p">)</span>
     <span class="c1"># Whether (one  of) the scheduler is scheduling this DAG at the moment</span>
     <span class="n">scheduler_lock</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Boolean</span><span class="p">)</span>
@@ -2567,7 +2684,7 @@
 
 
 <span class="nd">@functools.total_ordering</span>
-<div class="viewcode-block" id="DAG"><a class="viewcode-back" href="../../code.html#airflow.models.DAG">[docs]</a><span class="k">class</span> <span class="nc">DAG</span><span class="p">(</span><span class="n">LoggingMixin</span><span class="p">):</span>
+<div class="viewcode-block" id="DAG"><a class="viewcode-back" href="../../code.html#airflow.models.DAG">[docs]</a><span class="k">class</span> <span class="nc">DAG</span><span class="p">(</span><span class="n">BaseDag</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
     <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    A dag (directed acyclic graph) is a collection of tasks with directional</span>
 <span class="sd">    dependencies. A dag also has a schedule, a start end an end date</span>
@@ -2655,8 +2772,14 @@
             <span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_args</span><span class="p">[</span><span class="s1">&#39;params&#39;</span><span class="p">]</span>
 
         <span class="n">validate_key</span><span class="p">(</span><span class="n">dag_id</span><span class="p">)</span>
+
+        <span class="c1"># Properties from BaseDag</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_dag_id</span> <span class="o">=</span> <span class="n">dag_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_full_filepath</span> <span class="o">=</span> <span class="n">full_filepath</span> <span class="k">if</span> <span class="n">full_filepath</span> <span class="k">else</span> <span class="s1">&#39;&#39;</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_concurrency</span> <span class="o">=</span> <span class="n">concurrency</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_pickle_id</span> <span class="o">=</span> <span class="bp">None</span>
+
         <span class="bp">self</span><span class="o">.</span><span class="n">task_dict</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">=</span> <span class="n">dag_id</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">start_date</span> <span class="o">=</span> <span class="n">start_date</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">end_date</span> <span class="o">=</span> <span class="n">end_date</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">schedule_interval</span> <span class="o">=</span> <span class="n">schedule_interval</span>
@@ -2666,14 +2789,12 @@
             <span class="bp">self</span><span class="o">.</span><span class="n">_schedule_interval</span> <span class="o">=</span> <span class="bp">None</span>
         <span class="k">else</span><span class="p">:</span>
             <span class="bp">self</span><span class="o">.</span><span class="n">_schedule_interval</span> <span class="o">=</span> <span class="n">schedule_interval</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">full_filepath</span> <span class="o">=</span> <span class="n">full_filepath</span> <span class="k">if</span> <span class="n">full_filepath</span> <span class="k">else</span> <span class="s1">&#39;&#39;</span>
         <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">template_searchpath</span><span class="p">,</span> <span class="n">six</span><span class="o">.</span><span class="n">string_types</span><span class="p">):</span>
             <span class="n">template_searchpath</span> <span class="o">=</span> <span class="p">[</span><span class="n">template_searchpath</span><span class="p">]</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">template_searchpath</span> <span class="o">=</span> <span class="n">template_searchpath</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">parent_dag</span> <span class="o">=</span> <span class="bp">None</span>  <span class="c1"># Gets set when DAGs are loaded</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">last_loaded</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">safe_dag_id</span> <span class="o">=</span> <span class="n">dag_id</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">,</span> <span class="s1">&#39;__dot__&#39;</span><span class="p">)</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">concurrency</span> <span class="o">=</span> <span class="n">concurrency</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">max_active_runs</span> <span class="o">=</span> <span class="n">max_active_runs</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">dagrun_timeout</span> <span class="o">=</span> <span class="n">dagrun_timeout</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">sla_miss_callback</span> <span class="o">=</span> <span class="n">sla_miss_callback</span>
@@ -2695,10 +2816,12 @@
     <span class="k">def</span> <span class="nf">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
         <span class="k">return</span> <span class="p">(</span>
             <span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">==</span> <span class="nb">type</span><span class="p">(</span><span class="n">other</span><span class="p">)</span> <span class="ow">and</span>
-            <span class="nb">all</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">__dict__</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">c</span><span class="p">,</span> <span class="bp">None</span><span class="p">)</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">__dict__</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">c</span><span class="p">,</span> <span class="bp">None</span><span class="p">)</span>
+            <span class="c1"># Use getattr() instead of __dict__ as __dict__ doesn&#39;t return</span>
+            <span class="c1"># correct values for properties.</span>
+            <span class="nb">all</span><span class="p">(</span><span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">c</span><span class="p">,</span> <span class="bp">None</span><span class="p">)</span> <span class="o">==</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="n">c</span><span class="p">,</span> <span class="bp">None</span><span class="p">)</span>
                 <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_comps</span><span class="p">))</span>
 
-    <span class="k">def</span> <span class="nf">__neq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
+    <span class="k">def</span> <span class="nf">__ne__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
         <span class="k">return</span> <span class="ow">not</span> <span class="bp">self</span> <span class="o">==</span> <span class="n">other</span>
 
     <span class="k">def</span> <span class="nf">__lt__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
@@ -2770,6 +2893,38 @@
         <span class="k">return</span> <span class="n">dttm</span></div>
 
     <span class="nd">@property</span>
+    <span class="k">def</span> <span class="nf">dag_id</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_dag_id</span>
+
+    <span class="nd">@dag_id.setter</span>
+    <span class="k">def</span> <span class="nf">dag_id</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_dag_id</span> <span class="o">=</span> <span class="n">value</span>
+
+    <span class="nd">@property</span>
+    <span class="k">def</span> <span class="nf">full_filepath</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_full_filepath</span>
+
+    <span class="nd">@full_filepath.setter</span>
+    <span class="k">def</span> <span class="nf">full_filepath</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_full_filepath</span> <span class="o">=</span> <span class="n">value</span>
+
+    <span class="nd">@property</span>
+    <span class="k">def</span> <span class="nf">concurrency</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_concurrency</span>
+
+    <span class="nd">@concurrency.setter</span>
+    <span class="k">def</span> <span class="nf">concurrency</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_concurrency</span> <span class="o">=</span> <span class="n">value</span>
+
+    <span class="nd">@property</span>
+    <span class="k">def</span> <span class="nf">pickle_id</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_pickle_id</span>
+
+    <span class="nd">@pickle_id.setter</span>
+    <span class="k">def</span> <span class="nf">pickle_id</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_pickle_id</span> <span class="o">=</span> <span class="n">value</span>
+
+    <span class="nd">@property</span>
     <span class="k">def</span> <span class="nf">tasks</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
         <span class="k">return</span> <span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_dict</span><span class="o">.</span><span class="n">values</span><span class="p">())</span>
 
@@ -2866,6 +3021,15 @@
                 <span class="n">l</span> <span class="o">+=</span> <span class="n">task</span><span class="o">.</span><span class="n">subdag</span><span class="o">.</span><span class="n">subdags</span>
         <span class="k">return</span> <span class="n">l</span>
 
+    <span class="nd">@property</span>
+    <span class="k">def</span> <span class="nf">reached_max_runs</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="n">active_runs</span> <span class="o">=</span> <span class="n">DagRun</span><span class="o">.</span><span class="n">find</span><span class="p">(</span>
+            <span class="n">dag_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span>
+            <span class="n">state</span><span class="o">=</span><span class="n">State</span><span class="o">.</span><span class="n">RUNNING</span><span class="p">,</span>
+            <span class="n">external_trigger</span><span class="o">=</span><span class="bp">False</span>
+        <span class="p">)</span>
+        <span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="n">active_runs</span><span class="p">)</span> <span class="o">&gt;=</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_active_runs</span>
+
     <span class="k">def</span> <span class="nf">resolve_template_files</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
         <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">tasks</span><span class="p">:</span>
             <span class="n">t</span><span class="o">.</span><span class="n">resolve_template_files</span><span class="p">()</span>
@@ -2932,9 +3096,9 @@
         <span class="n">dates</span> <span class="o">=</span> <span class="n">utils_date_range</span><span class="p">(</span><span class="n">start_date</span><span class="p">,</span> <span class="n">end_date</span><span class="p">)</span>
         <span class="n">drs</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DagModel</span><span class="p">)</span><span class="o">.</span><span class="n">filter_by</span><span class="p">(</span><span class="n">dag_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">)</span><span class="o">.</span><span class="n">all</span><span class="p">()</span>
         <span class="k">for</span> <span class="n">dr</span> <span class="ow">in</span> <span class="n">drs</span><span class="p">:</span>
-            <span class="n">dr</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="n">State</span><span class="o">.</span><span class="n">RUNNING</span>
+            <span class="n">dr</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="n">state</span>
 
-    <span class="k">def</span> <span class="nf">clear</span><span class="p">(</span>
+<div class="viewcode-block" id="DAG.clear"><a class="viewcode-back" href="../../code.html#airflow.models.DAG.clear">[docs]</a>    <span class="k">def</span> <span class="nf">clear</span><span class="p">(</span>
             <span class="bp">self</span><span class="p">,</span> <span class="n">start_date</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">end_date</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
             <span class="n">only_failed</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
             <span class="n">only_running</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
@@ -2942,11 +3106,11 @@
             <span class="n">include_subdags</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span>
             <span class="n">reset_dag_runs</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span>
             <span class="n">dry_run</span><span class="o">=</span><span class="bp">False</span><span class="p">):</span>
-        <span class="n">session</span> <span class="o">=</span> <span class="n">settings</span><span class="o">.</span><span class="n">Session</span><span class="p">()</span>
         <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Clears a set of task instances associated with the current dag for</span>
 <span class="sd">        a specified date range.</span>
 <span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">session</span> <span class="o">=</span> <span class="n">settings</span><span class="o">.</span><span class="n">Session</span><span class="p">()</span>
         <span class="n">TI</span> <span class="o">=</span> <span class="n">TaskInstance</span>
         <span class="n">tis</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">TI</span><span class="p">)</span>
         <span class="k">if</span> <span class="n">include_subdags</span><span class="p">:</span>
@@ -2998,7 +3162,7 @@
 
         <span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
         <span class="n">session</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
-        <span class="k">return</span> <span class="n">count</span>
+        <span class="k">return</span> <span class="n">count</span></div>
 
     <span class="k">def</span> <span class="nf">__deepcopy__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">memo</span><span class="p">):</span>
         <span class="c1"># Swiwtcharoo to go around deepcopying objects coming through the</span>
@@ -3230,7 +3394,81 @@
         <span class="n">run</span><span class="o">.</span><span class="n">verify_integrity</span><span class="p">(</span><span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">)</span>
 
         <span class="n">run</span><span class="o">.</span><span class="n">refresh_from_db</span><span class="p">()</span>
-        <span class="k">return</span> <span class="n">run</span></div></div>
+        <span class="k">return</span> <span class="n">run</span></div>
+
+    <span class="nd">@staticmethod</span>
+    <span class="nd">@provide_session</span>
+<div class="viewcode-block" id="DAG.sync_to_db"><a class="viewcode-back" href="../../code.html#airflow.models.DAG.sync_to_db">[docs]</a>    <span class="k">def</span> <span class="nf">sync_to_db</span><span class="p">(</span><span class="n">dag</span><span class="p">,</span> <span class="n">owner</span><span class="p">,</span> <span class="n">sync_time</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Save attributes about this DAG to the DB. Note that this method</span>
+<span class="sd">        can be called for both DAGs and SubDAGs. A SubDag is actually a</span>
+<span class="sd">        SubDagOperator.</span>
+
+<span class="sd">        :param dag: the DAG object to save to the DB</span>
+<span class="sd">        :type dag: DAG</span>
+<span class="sd">        :own</span>
+<span class="sd">        :param sync_time: The time that the DAG should be marked as sync&#39;ed</span>
+<span class="sd">        :type sync_time: datetime</span>
+<span class="sd">        :return: None</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">orm_dag</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span>
+            <span class="n">DagModel</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DagModel</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="n">dag</span><span class="o">.</span><span class="n">dag_id</span><span class="p">)</span><span class="o">.</span><span class="n">first</span><span class="p">()</span>
+        <span class="k">if</span> <span class="ow">not</span> <span class="n">orm_dag</span><span class="p">:</span>
+            <span class="n">orm_dag</span> <span class="o">=</span> <span class="n">DagModel</span><span class="p">(</span><span class="n">dag_id</span><span class="o">=</span><span class="n">dag</span><span class="o">.</span><span class="n">dag_id</span><span class="p">)</span>
+            <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Creating ORM DAG for </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span>
+                         <span class="n">dag</span><span class="o">.</span><span class="n">dag_id</span><span class="p">)</span>
+        <span class="n">orm_dag</span><span class="o">.</span><span class="n">fileloc</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">full_filepath</span>
+        <span class="n">orm_dag</span><span class="o">.</span><span class="n">is_subdag</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">is_subdag</span>
+        <span class="n">orm_dag</span><span class="o">.</span><span class="n">owners</span> <span class="o">=</span> <span class="n">owner</span>
+        <span class="n">orm_dag</span><span class="o">.</span><span class="n">is_active</span> <span class="o">=</span> <span class="bp">True</span>
+        <span class="n">orm_dag</span><span class="o">.</span><span class="n">last_scheduler_run</span> <span class="o">=</span> <span class="n">sync_time</span>
+        <span class="n">session</span><span class="o">.</span><span class="n">merge</span><span class="p">(</span><span class="n">orm_dag</span><span class="p">)</span>
+        <span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
+
+        <span class="k">for</span> <span class="n">subdag</span> <span class="ow">in</span> <span class="n">dag</span><span class="o">.</span><span class="n">subdags</span><span class="p">:</span>
+            <span class="n">DAG</span><span class="o">.</span><span class="n">sync_to_db</span><span class="p">(</span><span class="n">subdag</span><span class="p">,</span> <span class="n">owner</span><span class="p">,</span> <span class="n">sync_time</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">)</span></div>
+
+    <span class="nd">@staticmethod</span>
+    <span class="nd">@provide_session</span>
+<div class="viewcode-block" id="DAG.deactivate_unknown_dags"><a class="viewcode-back" href="../../code.html#airflow.models.DAG.deactivate_unknown_dags">[docs]</a>    <span class="k">def</span> <span class="nf">deactivate_unknown_dags</span><span class="p">(</span><span class="n">active_dag_ids</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Given a list of known DAGs, deactivate any other DAGs that are</span>
+<span class="sd">        marked as active in the ORM</span>
+
+<span class="sd">        :param active_dag_ids: list of DAG IDs that are active</span>
+<span class="sd">        :type active_dag_ids: list[unicode]</span>
+<span class="sd">        :return: None</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+
+        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">active_dag_ids</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+            <span class="k">return</span>
+        <span class="k">for</span> <span class="n">dag</span> <span class="ow">in</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span>
+                <span class="n">DagModel</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="o">~</span><span class="n">DagModel</span><span class="o">.</span><span class="n">dag_id</span><span class="o">.</span><span class="n">in_</span><span class="p">(</span><span class="n">active_dag_ids</span><span class="p">))</span><span class="o">.</span><span

<TRUNCATED>

[3/7] incubator-airflow-site git commit: New version of the docs

[ma...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/code.html
----------------------------------------------------------------------
diff --git a/code.html b/code.html
index 592ec27..91192d3 100644
--- a/code.html
+++ b/code.html
@@ -187,7 +187,7 @@ to understand the primitive features that can be leveraged in your
 DAGs.</p>
 <dl class="class">
 <dt id="airflow.models.BaseOperator">
-<em class="property">class </em><code class="descclassname">airflow.models.</code><code class="descname">BaseOperator</code><span class="sig-paren">(</span><em>task_id</em>, <em>owner='airflow'</em>, <em>email=None</em>, <em>email_on_retry=True</em>, <em>email_on_failure=True</em>, <em>retries=0</em>, <em>retry_delay=datetime.timedelta(0</em>, <em>300)</em>, <em>start_date=None</em>, <em>end_date=None</em>, <em>schedule_interval=None</em>, <em>depends_on_past=False</em>, <em>wait_for_downstream=False</em>, <em>dag=None</em>, <em>params=None</em>, <em>default_args=None</em>, <em>adhoc=False</em>, <em>priority_weight=1</em>, <em>queue='default'</em>, <em>pool=None</em>, <em>sla=None</em>, <em>execution_timeout=None</em>, <em>on_failure_callback=None</em>, <em>on_success_callback=None</em>, <em>on_retry_callback=None</em>, <em>trigger_rule=u'all_success'</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/models.ht
 ml#BaseOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.BaseOperator" title="Permalink to this definition">�</a></dt>
+<em class="property">class </em><code class="descclassname">airflow.models.</code><code class="descname">BaseOperator</code><span class="sig-paren">(</span><em>task_id</em>, <em>owner='airflow'</em>, <em>email=None</em>, <em>email_on_retry=True</em>, <em>email_on_failure=True</em>, <em>retries=0</em>, <em>retry_delay=datetime.timedelta(0</em>, <em>300)</em>, <em>retry_exponential_backoff=False</em>, <em>max_retry_delay=None</em>, <em>start_date=None</em>, <em>end_date=None</em>, <em>schedule_interval=None</em>, <em>depends_on_past=False</em>, <em>wait_for_downstream=False</em>, <em>dag=None</em>, <em>params=None</em>, <em>default_args=None</em>, <em>adhoc=False</em>, <em>priority_weight=1</em>, <em>queue='default'</em>, <em>pool=None</em>, <em>sla=None</em>, <em>execution_timeout=None</em>, <em>on_failure_callback=None</em>, <em>on_success_callback=None</em>, <em>on_retry_callback=None</em>, <em>trigger_rule=u'all_success'</em>, <em>resources=None</em>, <em>*args</em>, <em>**kwargs<
 /em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/models.html#BaseOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.BaseOperator" title="Permalink to this definition">�</a></dt>
 <dd><p>Abstract base class for all operators. Since operators create objects that
 become node in the dag, BaseOperator contains many recursive methods for
 dag crawling behavior. To derive this class, you are expected to override
@@ -218,6 +218,10 @@ operators.</p>
 <li><strong>retries</strong> (<em>int</em>) &#8211; the number of retries that should be performed before
 failing the task</li>
 <li><strong>retry_delay</strong> (<em>timedelta</em>) &#8211; delay between retries</li>
+<li><strong>retry_exponential_backoff</strong> (<em>bool</em>) &#8211; allow progressive longer waits between
+retries by using exponential backoff algorithm on retry delay (delay
+will be converted into seconds)</li>
+<li><strong>max_retry_delay</strong> (<em>timedelta</em>) &#8211; maximum delay interval between retries</li>
 <li><strong>start_date</strong> (<em>datetime</em>) &#8211; The <code class="docutils literal"><span class="pre">start_date</span></code> for the task, determines
 the <code class="docutils literal"><span class="pre">execution_date</span></code> for the first task instance. The best practice
 is to have the start_date rounded
@@ -282,6 +286,8 @@ for the task to get triggered. Options are:
 default is <code class="docutils literal"><span class="pre">all_success</span></code>. Options can be set as string or
 using the constants defined in the static class
 <code class="docutils literal"><span class="pre">airflow.utils.TriggerRule</span></code></li>
+<li><strong>resources</strong> (<em>dict</em>) &#8211; A map of resource parameter names (the argument names of the
+Resources constructor) to their values.</li>
 </ul>
 </td>
 </tr>
@@ -323,14 +329,15 @@ between each tries</li>
 <div class="section" id="module-airflow.operators">
 <span id="operator-api"></span><h3>Operator API<a class="headerlink" href="#module-airflow.operators" title="Permalink to this headline">�</a></h3>
 <p>Importer that dynamically loads a class and module from its parent. This
-allows Airflow to support <cite>from airflow.operators.bash_operator import
-BashOperator</cite> even though BashOperator is actually in
-airflow.operators.bash_operator.</p>
+allows Airflow to support <code class="docutils literal"><span class="pre">from</span> <span class="pre">airflow.operators</span> <span class="pre">import</span> <span class="pre">BashOperator</span></code>
+even though BashOperator is actually in
+<code class="docutils literal"><span class="pre">airflow.operators.bash_operator</span></code>.</p>
 <p>The importer also takes over for the parent_module by wrapping it. This is
 required to support attribute-based usage:</p>
-<blockquote>
-<div>from airflow import operators
-operators.BashOperator(...)</div></blockquote>
+<div class="code python highlight-default"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">operators</span>
+<span class="n">operators</span><span class="o">.</span><span class="n">BashOperator</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
+</pre></div>
+</div>
 <dl class="class">
 <dt id="airflow.operators.BashOperator">
 <em class="property">class </em><code class="descclassname">airflow.operators.</code><code class="descname">BashOperator</code><span class="sig-paren">(</span><em>bash_command</em>, <em>xcom_push=False</em>, <em>env=None</em>, <em>output_encoding='utf-8'</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/bash_operator.html#BashOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.operators.BashOperator" title="Permalink to this definition">�</a></dt>
@@ -343,6 +350,8 @@ operators.BashOperator(...)</div></blockquote>
 <tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple">
 <li><strong>bash_command</strong> (<em>string</em>) &#8211; The command, set of commands or reference to a
 bash script (must be &#8216;.sh&#8217;) to be executed.</li>
+<li><strong>xcom_push</strong> (<em>bool</em>) &#8211; If xcom_push is True, the last line written to stdout
+will also be pushed to an XCom when the bash command completes.</li>
 <li><strong>env</strong> (<em>dict</em>) &#8211; If env is not None, it must be a mapping that defines the
 environment variables for the new process; these are used instead
 of inheriting the current process environment, which is the default
@@ -418,7 +427,7 @@ DAG.</p>
 
 <dl class="class">
 <dt id="airflow.operators.EmailOperator">
-<em class="property">class </em><code class="descclassname">airflow.operators.</code><code class="descname">EmailOperator</code><span class="sig-paren">(</span><em>to</em>, <em>subject</em>, <em>html_content</em>, <em>files=None</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/email_operator.html#EmailOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.operators.EmailOperator" title="Permalink to this definition">�</a></dt>
+<em class="property">class </em><code class="descclassname">airflow.operators.</code><code class="descname">EmailOperator</code><span class="sig-paren">(</span><em>to</em>, <em>subject</em>, <em>html_content</em>, <em>files=None</em>, <em>cc=None</em>, <em>bcc=None</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/email_operator.html#EmailOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.operators.EmailOperator" title="Permalink to this definition">�</a></dt>
 <dd><p>Bases: <a class="reference internal" href="#airflow.models.BaseOperator" title="airflow.models.BaseOperator"><code class="xref py py-class docutils literal"><span class="pre">airflow.models.BaseOperator</span></code></a></p>
 <p>Sends an email.</p>
 <table class="docutils field-list" frame="void" rules="none">
@@ -431,6 +440,8 @@ DAG.</p>
 <li><strong>html_content</strong> (<em>string</em>) &#8211; content of the email (templated), html markup
 is allowed</li>
 <li><strong>files</strong> (<em>list</em>) &#8211; file names to attach in email</li>
+<li><strong>cc</strong> (<em>list or string (comma or semicolon delimited)</em>) &#8211; list of recipients to be added in CC field</li>
+<li><strong>bcc</strong> (<em>list or string (comma or semicolon delimited)</em>) &#8211; list of recipients to be added in BCC field</li>
 </ul>
 </td>
 </tr>
@@ -525,7 +536,7 @@ results of the query as a csv to a Samba location.</p>
 
 <dl class="class">
 <dt id="airflow.operators.HiveOperator">
-<em class="property">class </em><code class="descclassname">airflow.operators.</code><code class="descname">HiveOperator</code><span class="sig-paren">(</span><em>hql</em>, <em>hive_cli_conn_id='hive_cli_default'</em>, <em>schema='default'</em>, <em>hiveconf_jinja_translate=False</em>, <em>script_begin_tag=None</em>, <em>run_as_owner=False</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/hive_operator.html#HiveOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.operators.HiveOperator" title="Permalink to this definition">�</a></dt>
+<em class="property">class </em><code class="descclassname">airflow.operators.</code><code class="descname">HiveOperator</code><span class="sig-paren">(</span><em>hql</em>, <em>hive_cli_conn_id='hive_cli_default'</em>, <em>schema='default'</em>, <em>hiveconf_jinja_translate=False</em>, <em>script_begin_tag=None</em>, <em>run_as_owner=False</em>, <em>mapred_queue=None</em>, <em>mapred_queue_priority=None</em>, <em>mapred_job_name=None</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/hive_operator.html#HiveOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.operators.HiveOperator" title="Permalink to this definition">�</a></dt>
 <dd><p>Bases: <a class="reference internal" href="#airflow.models.BaseOperator" title="airflow.models.BaseOperator"><code class="xref py py-class docutils literal"><span class="pre">airflow.models.BaseOperator</span></code></a></p>
 <p>Executes hql code in a specific Hive database.</p>
 <table class="docutils field-list" frame="void" rules="none">
@@ -542,6 +553,11 @@ you may want to use this along with the
 object documentation for more details.</li>
 <li><strong>script_begin_tag</strong> (<em>str</em>) &#8211; If defined, the operator will get rid of the
 part of the script before the first occurrence of <cite>script_begin_tag</cite></li>
+<li><strong>mapred_queue</strong> (<em>string</em>) &#8211; queue used by the Hadoop CapacityScheduler</li>
+<li><strong>mapred_queue_priority</strong> (<em>string</em>) &#8211; priority within CapacityScheduler queue.
+Possible settings include: VERY_HIGH, HIGH, NORMAL, LOW, VERY_LOW</li>
+<li><strong>mapred_job_name</strong> (<em>string</em>) &#8211; This name will appear in the jobtracker.
+This can make monitoring easier.</li>
 </ul>
 </td>
 </tr>
@@ -554,7 +570,7 @@ part of the script before the first occurrence of <cite>script_begin_tag</cite><
 <em class="property">class </em><code class="descclassname">airflow.operators.</code><code class="descname">HivePartitionSensor</code><span class="sig-paren">(</span><em>table</em>, <em>partition=&quot;ds='{{ ds }}'&quot;</em>, <em>metastore_conn_id='metastore_default'</em>, <em>schema='default'</em>, <em>poke_interval=180</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/sensors.html#HivePartitionSensor"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.operators.HivePartitionSensor" title="Permalink to this definition">�</a></dt>
 <dd><p>Bases: <a class="reference internal" href="#airflow.operators.sensors.BaseSensorOperator" title="airflow.operators.sensors.BaseSensorOperator"><code class="xref py py-class docutils literal"><span class="pre">sensors.BaseSensorOperator</span></code></a></p>
 <p>Waits for a partition to show up in Hive.</p>
-<p>Note: Because &#64;partition supports general logical operators, it
+<p>Note: Because <code class="docutils literal"><span class="pre">partition</span></code> supports general logical operators, it
 can be inefficient. Consider using NamedHivePartitionSensor instead if
 you don&#8217;t need the full flexibility of HivePartitionSensor.</p>
 <table class="docutils field-list" frame="void" rules="none">
@@ -565,9 +581,9 @@ you don&#8217;t need the full flexibility of HivePartitionSensor.</p>
 <li><strong>table</strong> (<em>string</em>) &#8211; The name of the table to wait for, supports the dot
 notation (my_database.my_table)</li>
 <li><strong>partition</strong> (<em>string</em>) &#8211; The partition clause to wait for. This is passed as
-is to the metastore Thrift client &#8220;get_partitions_by_filter&#8221; method,
-and apparently supports SQL like notation as in <cite>ds=&#8216;2015-01-01&#8217;
-AND type=&#8217;value&#8217;</cite> and &gt; &lt; sings as in &#8220;ds&gt;=2015-01-01&#8221;</li>
+is to the metastore Thrift client <code class="docutils literal"><span class="pre">get_partitions_by_filter</span></code> method,
+and apparently supports SQL like notation as in <code class="docutils literal"><span class="pre">ds='2015-01-01'</span>
+<span class="pre">AND</span> <span class="pre">type='value'</span></code> and comparison operators as in <code class="docutils literal"><span class="pre">&quot;ds&gt;=2015-01-01&quot;</span></code></li>
 <li><strong>metastore_conn_id</strong> (<em>str</em>) &#8211; reference to the metastore thrift service
 connection id</li>
 </ul>
@@ -835,6 +851,32 @@ and values</li>
 </dd></dl>
 
 <dl class="class">
+<dt id="airflow.operators.NamedHivePartitionSensor">
+<em class="property">class </em><code class="descclassname">airflow.operators.</code><code class="descname">NamedHivePartitionSensor</code><span class="sig-paren">(</span><em>partition_names</em>, <em>metastore_conn_id='metastore_default'</em>, <em>poke_interval=180</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/sensors.html#NamedHivePartitionSensor"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.operators.NamedHivePartitionSensor" title="Permalink to this definition">�</a></dt>
+<dd><p>Bases: <a class="reference internal" href="#airflow.operators.sensors.BaseSensorOperator" title="airflow.operators.sensors.BaseSensorOperator"><code class="xref py py-class docutils literal"><span class="pre">sensors.BaseSensorOperator</span></code></a></p>
+<p>Waits for a set of partitions to show up in Hive.</p>
+<table class="docutils field-list" frame="void" rules="none">
+<col class="field-name" />
+<col class="field-body" />
+<tbody valign="top">
+<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple">
+<li><strong>partition_names</strong> (<em>list of strings</em>) &#8211; List of fully qualified names of the
+partitions to wait for. A fully qualified name is of the
+form <code class="docutils literal"><span class="pre">schema.table/pk1=pv1/pk2=pv2</span></code>, for example,
+default.users/ds=2016-01-01. This is passed as is to the metastore
+Thrift client <code class="docutils literal"><span class="pre">get_partitions_by_name</span></code> method. Note that
+you cannot use logical or comparison operators as in
+HivePartitionSensor.</li>
+<li><strong>metastore_conn_id</strong> (<em>str</em>) &#8211; reference to the metastore thrift service
+connection id</li>
+</ul>
+</td>
+</tr>
+</tbody>
+</table>
+</dd></dl>
+
+<dl class="class">
 <dt id="airflow.operators.PostgresOperator">
 <em class="property">class </em><code class="descclassname">airflow.operators.</code><code class="descname">PostgresOperator</code><span class="sig-paren">(</span><em>sql</em>, <em>postgres_conn_id='postgres_default'</em>, <em>autocommit=False</em>, <em>parameters=None</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/postgres_operator.html#PostgresOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.operators.PostgresOperator" title="Permalink to this definition">�</a></dt>
 <dd><p>Bases: <a class="reference internal" href="#airflow.models.BaseOperator" title="airflow.models.BaseOperator"><code class="xref py py-class docutils literal"><span class="pre">airflow.models.BaseOperator</span></code></a></p>
@@ -863,12 +905,14 @@ Template reference are recognized by str ending in '.sql'</em>) &#8211; the sql
 a sql query that will return a single row. Each value on that
 first row is evaluated using python <code class="docutils literal"><span class="pre">bool</span></code> casting. If any of the
 values return <code class="docutils literal"><span class="pre">False</span></code> the check is failed and errors out.</p>
-<p>Note that Python bool casting evals the following as <code class="docutils literal"><span class="pre">False</span></code>:
-* False
-* 0
-* Empty string (<code class="docutils literal"><span class="pre">&quot;&quot;</span></code>)
-* Empty list (<code class="docutils literal"><span class="pre">[]</span></code>)
-* Empty dictionary or set (<code class="docutils literal"><span class="pre">{}</span></code>)</p>
+<p>Note that Python bool casting evals the following as <code class="docutils literal"><span class="pre">False</span></code>:</p>
+<ul class="simple">
+<li><code class="docutils literal"><span class="pre">False</span></code></li>
+<li><code class="docutils literal"><span class="pre">0</span></code></li>
+<li>Empty string (<code class="docutils literal"><span class="pre">&quot;&quot;</span></code>)</li>
+<li>Empty list (<code class="docutils literal"><span class="pre">[]</span></code>)</li>
+<li>Empty dictionary or set (<code class="docutils literal"><span class="pre">{}</span></code>)</li>
+</ul>
 <p>Given a query like <code class="docutils literal"><span class="pre">SELECT</span> <span class="pre">COUNT(*)</span> <span class="pre">FROM</span> <span class="pre">foo</span></code>, it will fail only if
 the count <code class="docutils literal"><span class="pre">==</span> <span class="pre">0</span></code>. You can craft much more complex query that could,
 for instance, check that the table has the same number of rows as
@@ -1206,14 +1250,15 @@ The default is False.</li>
 <div class="section" id="module-airflow.contrib.operators">
 <span id="community-contributed-operators"></span><h3>Community-contributed Operators<a class="headerlink" href="#module-airflow.contrib.operators" title="Permalink to this headline">�</a></h3>
 <p>Importer that dynamically loads a class and module from its parent. This
-allows Airflow to support <cite>from airflow.operators.bash_operator import
-BashOperator</cite> even though BashOperator is actually in
-airflow.operators.bash_operator.</p>
+allows Airflow to support <code class="docutils literal"><span class="pre">from</span> <span class="pre">airflow.operators</span> <span class="pre">import</span> <span class="pre">BashOperator</span></code>
+even though BashOperator is actually in
+<code class="docutils literal"><span class="pre">airflow.operators.bash_operator</span></code>.</p>
 <p>The importer also takes over for the parent_module by wrapping it. This is
 required to support attribute-based usage:</p>
-<blockquote>
-<div>from airflow import operators
-operators.BashOperator(...)</div></blockquote>
+<div class="code python highlight-default"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">operators</span>
+<span class="n">operators</span><span class="o">.</span><span class="n">BashOperator</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
+</pre></div>
+</div>
 <dl class="class">
 <dt id="airflow.contrib.operators.SSHExecuteOperator">
 <em class="property">class </em><code class="descclassname">airflow.contrib.operators.</code><code class="descname">SSHExecuteOperator</code><span class="sig-paren">(</span><em>ssh_hook</em>, <em>bash_command</em>, <em>xcom_push=False</em>, <em>env=None</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/ssh_execute_operator.html#SSHExecuteOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.SSHExecuteOperator" title="Permalink to this definition">�</a></dt>
@@ -1848,7 +1893,7 @@ persisted in the database.</p>
 <span class="target" id="module-airflow.models"></span><dl class="class">
 <dt id="airflow.models.DAG">
 <em class="property">class </em><code class="descclassname">airflow.models.</code><code class="descname">DAG</code><span class="sig-paren">(</span><em>dag_id</em>, <em>schedule_interval=datetime.timedelta(1)</em>, <em>start_date=None</em>, <em>end_date=None</em>, <em>full_filepath=None</em>, <em>template_searchpath=None</em>, <em>user_defined_macros=None</em>, <em>default_args=None</em>, <em>concurrency=16</em>, <em>max_active_runs=16</em>, <em>dagrun_timeout=None</em>, <em>sla_miss_callback=None</em>, <em>params=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/models.html#DAG"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.DAG" title="Permalink to this definition">�</a></dt>
-<dd><p>Bases: <code class="xref py py-class docutils literal"><span class="pre">airflow.utils.logging.LoggingMixin</span></code></p>
+<dd><p>Bases: <code class="xref py py-class docutils literal"><span class="pre">airflow.dag.base_dag.BaseDag</span></code>, <code class="xref py py-class docutils literal"><span class="pre">airflow.utils.logging.LoggingMixin</span></code></p>
 <p>A dag (directed acyclic graph) is a collection of tasks with directional
 dependencies. A dag also has a schedule, a start end an end date
 (optional). For each schedule, (say daily or hourly), the DAG needs to run
@@ -1933,6 +1978,13 @@ timeouts.</li>
 </dd></dl>
 
 <dl class="method">
+<dt id="airflow.models.DAG.clear">
+<code class="descname">clear</code><span class="sig-paren">(</span><em>start_date=None</em>, <em>end_date=None</em>, <em>only_failed=False</em>, <em>only_running=False</em>, <em>confirm_prompt=False</em>, <em>include_subdags=True</em>, <em>reset_dag_runs=True</em>, <em>dry_run=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/models.html#DAG.clear"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.DAG.clear" title="Permalink to this definition">�</a></dt>
+<dd><p>Clears a set of task instances associated with the current dag for
+a specified date range.</p>
+</dd></dl>
+
+<dl class="method">
 <dt id="airflow.models.DAG.cli">
 <code class="descname">cli</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/models.html#DAG.cli"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.DAG.cli" title="Permalink to this definition">�</a></dt>
 <dd><p>Exposes a CLI specific to this DAG</p>
@@ -1972,6 +2024,41 @@ run.
 :type session: Session</p>
 </dd></dl>
 
+<dl class="staticmethod">
+<dt id="airflow.models.DAG.deactivate_stale_dags">
+<em class="property">static </em><code class="descname">deactivate_stale_dags</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/models.html#DAG.deactivate_stale_dags"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.DAG.deactivate_stale_dags" title="Permalink to this definition">�</a></dt>
+<dd><p>Deactivate any DAGs that were last touched by the scheduler before
+the expiration date. These DAGs were likely deleted.</p>
+<table class="docutils field-list" frame="void" rules="none">
+<col class="field-name" />
+<col class="field-body" />
+<tbody valign="top">
+<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>expiration_date</strong> &#8211; set inactive DAGs that were touched before this</td>
+</tr>
+</tbody>
+</table>
+<p>time
+:type expiration_date: datetime
+:return: None</p>
+</dd></dl>
+
+<dl class="staticmethod">
+<dt id="airflow.models.DAG.deactivate_unknown_dags">
+<em class="property">static </em><code class="descname">deactivate_unknown_dags</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/models.html#DAG.deactivate_unknown_dags"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.DAG.deactivate_unknown_dags" title="Permalink to this definition">�</a></dt>
+<dd><p>Given a list of known DAGs, deactivate any other DAGs that are
+marked as active in the ORM</p>
+<table class="docutils field-list" frame="void" rules="none">
+<col class="field-name" />
+<col class="field-body" />
+<tbody valign="top">
+<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>active_dag_ids</strong> (<em>list[unicode]</em>) &#8211; list of DAG IDs that are active</td>
+</tr>
+<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body">None</td>
+</tr>
+</tbody>
+</table>
+</dd></dl>
+
 <dl class="attribute">
 <dt id="airflow.models.DAG.filepath">
 <code class="descname">filepath</code><a class="headerlink" href="#airflow.models.DAG.filepath" title="Permalink to this definition">�</a></dt>
@@ -2036,6 +2123,26 @@ upstream and downstream neighbours based on the flag passed.</p>
 <dd><p>Returns a list of the subdag objects associated to this DAG</p>
 </dd></dl>
 
+<dl class="staticmethod">
+<dt id="airflow.models.DAG.sync_to_db">
+<em class="property">static </em><code class="descname">sync_to_db</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/models.html#DAG.sync_to_db"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.DAG.sync_to_db" title="Permalink to this definition">�</a></dt>
+<dd><p>Save attributes about this DAG to the DB. Note that this method
+can be called for both DAGs and SubDAGs. A SubDag is actually a
+SubDagOperator.</p>
+<table class="docutils field-list" frame="void" rules="none">
+<col class="field-name" />
+<col class="field-body" />
+<tbody valign="top">
+<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>dag</strong> (<a class="reference internal" href="#airflow.models.DAG" title="airflow.models.DAG"><em>DAG</em></a>) &#8211; the DAG object to save to the DB</td>
+</tr>
+</tbody>
+</table>
+<p>:own
+:param sync_time: The time that the DAG should be marked as sync&#8217;ed
+:type sync_time: datetime
+:return: None</p>
+</dd></dl>
+
 <dl class="method">
 <dt id="airflow.models.DAG.tree_view">
 <code class="descname">tree_view</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/models.html#DAG.tree_view"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.DAG.tree_view" title="Permalink to this definition">�</a></dt>
@@ -2046,7 +2153,7 @@ upstream and downstream neighbours based on the flag passed.</p>
 
 <dl class="class">
 <dt>
-<em class="property">class </em><code class="descclassname">airflow.models.</code><code class="descname">BaseOperator</code><span class="sig-paren">(</span><em>task_id</em>, <em>owner='airflow'</em>, <em>email=None</em>, <em>email_on_retry=True</em>, <em>email_on_failure=True</em>, <em>retries=0</em>, <em>retry_delay=datetime.timedelta(0</em>, <em>300)</em>, <em>start_date=None</em>, <em>end_date=None</em>, <em>schedule_interval=None</em>, <em>depends_on_past=False</em>, <em>wait_for_downstream=False</em>, <em>dag=None</em>, <em>params=None</em>, <em>default_args=None</em>, <em>adhoc=False</em>, <em>priority_weight=1</em>, <em>queue='default'</em>, <em>pool=None</em>, <em>sla=None</em>, <em>execution_timeout=None</em>, <em>on_failure_callback=None</em>, <em>on_success_callback=None</em>, <em>on_retry_callback=None</em>, <em>trigger_rule=u'all_success'</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/models.ht
 ml#BaseOperator"><span class="viewcode-link">[source]</span></a></dt>
+<em class="property">class </em><code class="descclassname">airflow.models.</code><code class="descname">BaseOperator</code><span class="sig-paren">(</span><em>task_id</em>, <em>owner='airflow'</em>, <em>email=None</em>, <em>email_on_retry=True</em>, <em>email_on_failure=True</em>, <em>retries=0</em>, <em>retry_delay=datetime.timedelta(0</em>, <em>300)</em>, <em>retry_exponential_backoff=False</em>, <em>max_retry_delay=None</em>, <em>start_date=None</em>, <em>end_date=None</em>, <em>schedule_interval=None</em>, <em>depends_on_past=False</em>, <em>wait_for_downstream=False</em>, <em>dag=None</em>, <em>params=None</em>, <em>default_args=None</em>, <em>adhoc=False</em>, <em>priority_weight=1</em>, <em>queue='default'</em>, <em>pool=None</em>, <em>sla=None</em>, <em>execution_timeout=None</em>, <em>on_failure_callback=None</em>, <em>on_success_callback=None</em>, <em>on_retry_callback=None</em>, <em>trigger_rule=u'all_success'</em>, <em>resources=None</em>, <em>*args</em>, <em>**kwargs<
 /em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/models.html#BaseOperator"><span class="viewcode-link">[source]</span></a></dt>
 <dd><p>Bases: <code class="xref py py-class docutils literal"><span class="pre">future.types.newobject.newobject</span></code></p>
 <p>Abstract base class for all operators. Since operators create objects that
 become node in the dag, BaseOperator contains many recursive methods for
@@ -2078,6 +2185,10 @@ operators.</p>
 <li><strong>retries</strong> (<em>int</em>) &#8211; the number of retries that should be performed before
 failing the task</li>
 <li><strong>retry_delay</strong> (<em>timedelta</em>) &#8211; delay between retries</li>
+<li><strong>retry_exponential_backoff</strong> (<em>bool</em>) &#8211; allow progressive longer waits between
+retries by using exponential backoff algorithm on retry delay (delay
+will be converted into seconds)</li>
+<li><strong>max_retry_delay</strong> (<em>timedelta</em>) &#8211; maximum delay interval between retries</li>
 <li><strong>start_date</strong> (<em>datetime</em>) &#8211; The <code class="docutils literal"><span class="pre">start_date</span></code> for the task, determines
 the <code class="docutils literal"><span class="pre">execution_date</span></code> for the first task instance. The best practice
 is to have the start_date rounded
@@ -2142,6 +2253,8 @@ for the task to get triggered. Options are:
 default is <code class="docutils literal"><span class="pre">all_success</span></code>. Options can be set as string or
 using the constants defined in the static class
 <code class="docutils literal"><span class="pre">airflow.utils.TriggerRule</span></code></li>
+<li><strong>resources</strong> (<em>dict</em>) &#8211; A map of resource parameter names (the argument names of the
+Resources constructor) to their values.</li>
 </ul>
 </td>
 </tr>
@@ -2407,6 +2520,45 @@ path to add the feature</li>
 </table>
 </dd></dl>
 
+<dl class="staticmethod">
+<dt id="airflow.models.TaskInstance.generate_command">
+<em class="property">static </em><code class="descname">generate_command</code><span class="sig-paren">(</span><em>dag_id</em>, <em>task_id</em>, <em>execution_date</em>, <em>mark_success=False</em>, <em>ignore_dependencies=False</em>, <em>ignore_depends_on_past=False</em>, <em>force=False</em>, <em>local=False</em>, <em>pickle_id=None</em>, <em>file_path=None</em>, <em>raw=False</em>, <em>job_id=None</em>, <em>pool=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/models.html#TaskInstance.generate_command"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.TaskInstance.generate_command" title="Permalink to this definition">�</a></dt>
+<dd><p>Generates the shell command required to execute this task instance.</p>
+<table class="docutils field-list" frame="void" rules="none">
+<col class="field-name" />
+<col class="field-body" />
+<tbody valign="top">
+<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple">
+<li><strong>dag_id</strong> (<em>unicode</em>) &#8211; DAG ID</li>
+<li><strong>task_id</strong> (<em>unicode</em>) &#8211; Task ID</li>
+<li><strong>execution_date</strong> (<em>datetime</em>) &#8211; Execution date for the task</li>
+<li><strong>mark_success</strong> (<em>bool</em>) &#8211; Whether to mark the task as successful</li>
+<li><strong>ignore_dependencies</strong> &#8211; Whether to ignore the dependencies and run</li>
+</ul>
+</td>
+</tr>
+</tbody>
+</table>
+<p>anyway
+:type ignore_dependencies: bool
+:param ignore_depends_on_past: Whether to ignore the depends on past
+setting and run anyway
+:type ignore_depends_on_past: bool
+:param force: Whether to force running - see TaskInstance.run()
+:type force: bool
+:param local: Whether to run the task locally
+:type local: bool
+:param pickle_id: If the DAG was serialized to the DB, the ID
+associated with the pickled DAG
+:type pickle_id: unicode
+:param file_path: path to the file containing the DAG definition
+:param raw: raw mode (needs more details)
+:param job_id: job ID (needs more details)
+:param pool: the Airflow pool that the task should run in
+:type pool: unicode
+:return: shell command that can be used to run the task instance</p>
+</dd></dl>
+
 <dl class="method">
 <dt id="airflow.models.TaskInstance.is_premature">
 <code class="descname">is_premature</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/models.html#TaskInstance.is_premature"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.TaskInstance.is_premature" title="Permalink to this definition">�</a></dt>
@@ -2470,6 +2622,13 @@ dependencies. Defaults to False.</li>
 </dd></dl>
 
 <dl class="method">
+<dt id="airflow.models.TaskInstance.next_retry_datetime">
+<code class="descname">next_retry_datetime</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/models.html#TaskInstance.next_retry_datetime"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.TaskInstance.next_retry_datetime" title="Permalink to this definition">�</a></dt>
+<dd><p>Get datetime of the next retry if the task instance fails. For exponential
+backoff, retry_delay is used as base and will be converted to seconds.</p>
+</dd></dl>
+
+<dl class="method">
 <dt id="airflow.models.TaskInstance.pool_full">
 <code class="descname">pool_full</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/models.html#TaskInstance.pool_full"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.TaskInstance.pool_full" title="Permalink to this definition">�</a></dt>
 <dd><p>Returns a boolean as to whether the slot pool has room for this
@@ -2584,8 +2743,8 @@ task on a future date without it being immediately visible.</li>
 
 <dl class="class">
 <dt id="airflow.models.DagBag">
-<em class="property">class </em><code class="descclassname">airflow.models.</code><code class="descname">DagBag</code><span class="sig-paren">(</span><em>dag_folder=None</em>, <em>executor=&lt;airflow.executors.local_executor.LocalExecutor object&gt;</em>, <em>include_examples=True</em>, <em>sync_to_db=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/models.html#DagBag"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.DagBag" title="Permalink to this definition">�</a></dt>
-<dd><p>Bases: <code class="xref py py-class docutils literal"><span class="pre">airflow.utils.logging.LoggingMixin</span></code></p>
+<em class="property">class </em><code class="descclassname">airflow.models.</code><code class="descname">DagBag</code><span class="sig-paren">(</span><em>dag_folder=None</em>, <em>executor=&lt;airflow.executors.local_executor.LocalExecutor object&gt;</em>, <em>include_examples=True</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/models.html#DagBag"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.DagBag" title="Permalink to this definition">�</a></dt>
+<dd><p>Bases: <code class="xref py py-class docutils literal"><span class="pre">airflow.dag.base_dag.BaseDagBag</span></code>, <code class="xref py py-class docutils literal"><span class="pre">airflow.utils.logging.LoggingMixin</span></code></p>
 <p>A dagbag is a collection of dags, parsed out of a folder tree and has high
 level configuration settings, like what database to use as a backend and
 what executor to use to fire off tasks. This makes it easier to run
@@ -2598,7 +2757,7 @@ independent settings sets.</p>
 <col class="field-body" />
 <tbody valign="top">
 <tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple">
-<li><strong>dag_folder</strong> (<em>str</em>) &#8211; the folder to scan to find DAGs</li>
+<li><strong>dag_folder</strong> (<em>unicode</em>) &#8211; the folder to scan to find DAGs</li>
 <li><strong>executor</strong> &#8211; the executor to use when executing task instances
 in this DagBag</li>
 <li><strong>include_examples</strong> (<em>bool</em>) &#8211; whether to include the examples that ship
@@ -2688,14 +2847,15 @@ passwords when using operators or hooks.</p>
 <div class="section" id="module-airflow.hooks">
 <span id="hooks"></span><h2>Hooks<a class="headerlink" href="#module-airflow.hooks" title="Permalink to this headline">�</a></h2>
 <p>Importer that dynamically loads a class and module from its parent. This
-allows Airflow to support <cite>from airflow.operators.bash_operator import
-BashOperator</cite> even though BashOperator is actually in
-airflow.operators.bash_operator.</p>
+allows Airflow to support <code class="docutils literal"><span class="pre">from</span> <span class="pre">airflow.operators</span> <span class="pre">import</span> <span class="pre">BashOperator</span></code>
+even though BashOperator is actually in
+<code class="docutils literal"><span class="pre">airflow.operators.bash_operator</span></code>.</p>
 <p>The importer also takes over for the parent_module by wrapping it. This is
 required to support attribute-based usage:</p>
-<blockquote>
-<div>from airflow import operators
-operators.BashOperator(...)</div></blockquote>
+<div class="code python highlight-default"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">operators</span>
+<span class="n">operators</span><span class="o">.</span><span class="n">BashOperator</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
+</pre></div>
+</div>
 <dl class="class">
 <dt id="airflow.hooks.DbApiHook">
 <em class="property">class </em><code class="descclassname">airflow.hooks.</code><code class="descname">DbApiHook</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/dbapi_hook.html#DbApiHook"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.hooks.DbApiHook" title="Permalink to this definition">�</a></dt>
@@ -2855,7 +3015,7 @@ before executing the query.</li>
 
 <dl class="class">
 <dt id="airflow.hooks.HiveCliHook">
-<em class="property">class </em><code class="descclassname">airflow.hooks.</code><code class="descname">HiveCliHook</code><span class="sig-paren">(</span><em>hive_cli_conn_id='hive_cli_default'</em>, <em>run_as=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/hive_hooks.html#HiveCliHook"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.hooks.HiveCliHook" title="Permalink to this definition">�</a></dt>
+<em class="property">class </em><code class="descclassname">airflow.hooks.</code><code class="descname">HiveCliHook</code><span class="sig-paren">(</span><em>hive_cli_conn_id='hive_cli_default'</em>, <em>run_as=None</em>, <em>mapred_queue=None</em>, <em>mapred_queue_priority=None</em>, <em>mapred_job_name=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/hive_hooks.html#HiveCliHook"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.hooks.HiveCliHook" title="Permalink to this definition">�</a></dt>
 <dd><p>Bases: <code class="xref py py-class docutils literal"><span class="pre">airflow.hooks.base_hook.BaseHook</span></code></p>
 <p>Simple wrapper around the hive CLI.</p>
 <p>It also supports the <code class="docutils literal"><span class="pre">beeline</span></code>
@@ -2868,6 +3028,21 @@ extra field of your connection as in <code class="docutils literal"><span class=
 Parameters passed here can be overridden by run_cli&#8217;s hive_conf param</p>
 <p>The extra connection parameter <code class="docutils literal"><span class="pre">auth</span></code> gets passed as in the <code class="docutils literal"><span class="pre">jdbc</span></code>
 connection string as is.</p>
+<table class="docutils field-list" frame="void" rules="none">
+<col class="field-name" />
+<col class="field-body" />
+<tbody valign="top">
+<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple">
+<li><strong>mapred_queue</strong> (<em>string</em>) &#8211; queue used by the Hadoop Scheduler (Capacity or Fair)</li>
+<li><strong>mapred_queue_priority</strong> (<em>string</em>) &#8211; priority within the job queue.
+Possible settings include: VERY_HIGH, HIGH, NORMAL, LOW, VERY_LOW</li>
+<li><strong>mapred_job_name</strong> (<em>string</em>) &#8211; This name will appear in the jobtracker.
+This can make monitoring easier.</li>
+</ul>
+</td>
+</tr>
+</tbody>
+</table>
 <dl class="method">
 <dt id="airflow.hooks.HiveCliHook.load_file">
 <code class="descname">load_file</code><span class="sig-paren">(</span><em>filepath</em>, <em>table</em>, <em>delimiter='</em>, <em>'</em>, <em>field_dict=None</em>, <em>create=True</em>, <em>overwrite=True</em>, <em>partition=None</em>, <em>recreate=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/hive_hooks.html#HiveCliHook.load_file"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.hooks.HiveCliHook.load_file" title="Permalink to this definition">�</a></dt>
@@ -2901,16 +3076,17 @@ and values</li>
 <dl class="method">
 <dt id="airflow.hooks.HiveCliHook.run_cli">
 <code class="descname">run_cli</code><span class="sig-paren">(</span><em>hql</em>, <em>schema=None</em>, <em>verbose=True</em>, <em>hive_conf=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/hive_hooks.html#HiveCliHook.run_cli"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.hooks.HiveCliHook.run_cli" title="Permalink to this definition">�</a></dt>
-<dd><p>Run an hql statement using the hive cli. If hive_conf is specified it should be a
-dict and the entries will be set as key/value pairs in HiveConf</p>
+<dd><p>Run an hql statement using the hive cli. If hive_conf is specified
+it should be a dict and the entries will be set as key/value pairs
+in HiveConf</p>
 <table class="docutils field-list" frame="void" rules="none">
 <col class="field-name" />
 <col class="field-body" />
 <tbody valign="top">
-<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>hive_conf</strong> (<em>dict</em>) &#8211; if specified these key value pairs will be passed to hive as
-<code class="docutils literal"><span class="pre">-hiveconf</span> <span class="pre">&quot;key&quot;=&quot;value&quot;</span></code>. Note that they will be passed after the
-<code class="docutils literal"><span class="pre">hive_cli_params</span></code> and thus will override whatever values are specified in
-the database.</td>
+<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>hive_conf</strong> (<em>dict</em>) &#8211; if specified these key value pairs will be passed
+to hive as <code class="docutils literal"><span class="pre">-hiveconf</span> <span class="pre">&quot;key&quot;=&quot;value&quot;</span></code>. Note that they will be
+passed after the <code class="docutils literal"><span class="pre">hive_cli_params</span></code> and thus will override
+whatever values are specified in the database.</td>
 </tr>
 </tbody>
 </table>
@@ -3168,9 +3344,19 @@ checking for the result</p>
 <dl class="method">
 <dt id="airflow.hooks.DruidHook.load_from_hdfs">
 <code class="descname">load_from_hdfs</code><span class="sig-paren">(</span><em>datasource</em>, <em>static_path</em>, <em>ts_dim</em>, <em>columns</em>, <em>intervals</em>, <em>num_shards</em>, <em>target_partition_size</em>, <em>metric_spec=None</em>, <em>hadoop_dependency_coordinates=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/druid_hook.html#DruidHook.load_from_hdfs"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.hooks.DruidHook.load_from_hdfs" title="Permalink to this definition">�</a></dt>
-<dd><p>load data to druid from hdfs
-:params ts_dim: The column name to use as a timestamp
-:params metric_spec: A list of dictionaries</p>
+<dd><p>load data to druid from hdfs</p>
+<table class="docutils field-list" frame="void" rules="none">
+<col class="field-name" />
+<col class="field-body" />
+<tbody valign="top">
+<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple">
+<li><strong>ts_dim</strong> &#8211; The column name to use as a timestamp</li>
+<li><strong>metric_spec</strong> &#8211; A list of dictionaries</li>
+</ul>
+</td>
+</tr>
+</tbody>
+</table>
 </dd></dl>
 
 </dd></dl>
@@ -3507,14 +3693,15 @@ directory, files will be uploaded inside.</li>
 <div class="section" id="module-airflow.contrib.hooks">
 <span id="community-contributed-hooks"></span><h3>Community contributed hooks<a class="headerlink" href="#module-airflow.contrib.hooks" title="Permalink to this headline">�</a></h3>
 <p>Importer that dynamically loads a class and module from its parent. This
-allows Airflow to support <cite>from airflow.operators.bash_operator import
-BashOperator</cite> even though BashOperator is actually in
-airflow.operators.bash_operator.</p>
+allows Airflow to support <code class="docutils literal"><span class="pre">from</span> <span class="pre">airflow.operators</span> <span class="pre">import</span> <span class="pre">BashOperator</span></code>
+even though BashOperator is actually in
+<code class="docutils literal"><span class="pre">airflow.operators.bash_operator</span></code>.</p>
 <p>The importer also takes over for the parent_module by wrapping it. This is
 required to support attribute-based usage:</p>
-<blockquote>
-<div>from airflow import operators
-operators.BashOperator(...)</div></blockquote>
+<div class="code python highlight-default"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">operators</span>
+<span class="n">operators</span><span class="o">.</span><span class="n">BashOperator</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
+</pre></div>
+</div>
 <dl class="class">
 <dt id="airflow.contrib.hooks.BigQueryHook">
 <em class="property">class </em><code class="descclassname">airflow.contrib.hooks.</code><code class="descname">BigQueryHook</code><span class="sig-paren">(</span><em>bigquery_conn_id='bigquery_default'</em>, <em>delegate_to=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/bigquery_hook.html#BigQueryHook"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.BigQueryHook" title="Permalink to this definition">�</a></dt>
@@ -3719,6 +3906,24 @@ on the remote system (where the MLSD command is supported).</p>
 </dd></dl>
 
 <dl class="method">
+<dt id="airflow.contrib.hooks.FTPHook.rename">
+<code class="descname">rename</code><span class="sig-paren">(</span><em>from_name</em>, <em>to_name</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/ftp_hook.html#FTPHook.rename"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.FTPHook.rename" title="Permalink to this definition">�</a></dt>
+<dd><p>Rename a file.</p>
+<table class="docutils field-list" frame="void" rules="none">
+<col class="field-name" />
+<col class="field-body" />
+<tbody valign="top">
+<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple">
+<li><strong>from_name</strong> &#8211; rename file from name</li>
+<li><strong>to_name</strong> &#8211; rename file to name</li>
+</ul>
+</td>
+</tr>
+</tbody>
+</table>
+</dd></dl>
+
+<dl class="method">
 <dt id="airflow.contrib.hooks.FTPHook.retrieve_file">
 <code class="descname">retrieve_file</code><span class="sig-paren">(</span><em>remote_full_path</em>, <em>local_full_path_or_buffer</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/ftp_hook.html#FTPHook.retrieve_file"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.FTPHook.retrieve_file" title="Permalink to this definition">�</a></dt>
 <dd><p>Transfers the remote file to a local location.</p>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/faq.html
----------------------------------------------------------------------
diff --git a/faq.html b/faq.html
index 3d8cb5d..623e37f 100644
--- a/faq.html
+++ b/faq.html
@@ -91,7 +91,15 @@
 <li class="toctree-l1"><a class="reference internal" href="scheduler.html">Scheduling &amp; Triggers</a></li>
 <li class="toctree-l1"><a class="reference internal" href="plugins.html">Plugins</a></li>
 <li class="toctree-l1"><a class="reference internal" href="security.html">Security</a></li>
-<li class="toctree-l1 current"><a class="current reference internal" href="#">FAQ</a></li>
+<li class="toctree-l1 current"><a class="current reference internal" href="#">FAQ</a><ul>
+<li class="toctree-l2"><a class="reference internal" href="#why-isn-t-my-task-getting-scheduled">Why isn&#8217;t my task getting scheduled?</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#how-do-i-trigger-tasks-based-on-another-task-s-failure">How do I trigger tasks based on another task&#8217;s failure?</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#why-are-connection-passwords-still-not-encrypted-in-the-metadata-db-after-i-installed-airflow-crypto">Why are connection passwords still not encrypted in the metadata db after I installed airflow[crypto]?</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#what-s-the-deal-with-start-date">What&#8217;s the deal with <code class="docutils literal"><span class="pre">start_date</span></code>?</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#how-can-i-create-dags-dynamically">How can I create DAGs dynamically?</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#what-are-all-the-airflow-run-commands-in-my-process-list">What are all the <code class="docutils literal"><span class="pre">airflow</span> <span class="pre">run</span></code> commands in my process list?</a></li>
+</ul>
+</li>
 <li class="toctree-l1"><a class="reference internal" href="code.html">API Reference</a></li>
 </ul>
 
@@ -139,7 +147,8 @@
             
   <div class="section" id="faq">
 <h1>FAQ<a class="headerlink" href="#faq" title="Permalink to this headline">�</a></h1>
-<p><strong>Why isn&#8217;t my task getting scheduled?</strong></p>
+<div class="section" id="why-isn-t-my-task-getting-scheduled">
+<h2>Why isn&#8217;t my task getting scheduled?<a class="headerlink" href="#why-isn-t-my-task-getting-scheduled" title="Permalink to this headline">�</a></h2>
 <p>There are very many reasons why your task might not be getting scheduled.
 Here are some of the common causes:</p>
 <ul class="simple">
@@ -180,15 +189,21 @@ how many <code class="docutils literal"><span class="pre">running</span></code>
 </ul>
 <p>You may also want to read the Scheduler section of the docs and make
 sure you fully understand how it proceeds.</p>
-<p><strong>How do I trigger tasks based on another task&#8217;s failure?</strong></p>
+</div>
+<div class="section" id="how-do-i-trigger-tasks-based-on-another-task-s-failure">
+<h2>How do I trigger tasks based on another task&#8217;s failure?<a class="headerlink" href="#how-do-i-trigger-tasks-based-on-another-task-s-failure" title="Permalink to this headline">�</a></h2>
 <p>Check out the <code class="docutils literal"><span class="pre">Trigger</span> <span class="pre">Rule</span></code> section in the Concepts section of the
 documentation</p>
-<p><strong>Why are connection passwords still not encrypted in the metadata db after I installed airflow[crypto]</strong>?</p>
+</div>
+<div class="section" id="why-are-connection-passwords-still-not-encrypted-in-the-metadata-db-after-i-installed-airflow-crypto">
+<h2>Why are connection passwords still not encrypted in the metadata db after I installed airflow[crypto]?<a class="headerlink" href="#why-are-connection-passwords-still-not-encrypted-in-the-metadata-db-after-i-installed-airflow-crypto" title="Permalink to this headline">�</a></h2>
 <ul class="simple">
 <li>Verify that the <code class="docutils literal"><span class="pre">fernet_key</span></code> defined in <code class="docutils literal"><span class="pre">$AIRFLOW_HOME/airflow.cfg</span></code> is a valid Fernet key. It must be a base64-encoded 32-byte key. You need to restart the webserver after you update the key</li>
 <li>For existing connections (the ones that you had defined before installing <code class="docutils literal"><span class="pre">airflow[crypto]</span></code> and creating a Fernet key), you need to open each connection in the connection admin UI, re-type the password, and save it</li>
 </ul>
-<p><strong>What&#8217;s the deal with ``start_date``?</strong></p>
+</div>
+<div class="section" id="what-s-the-deal-with-start-date">
+<h2>What&#8217;s the deal with <code class="docutils literal"><span class="pre">start_date</span></code>?<a class="headerlink" href="#what-s-the-deal-with-start-date" title="Permalink to this headline">�</a></h2>
 <p><code class="docutils literal"><span class="pre">start_date</span></code> is partly legacy from the pre-DagRun era, but it is still
 relevant in many ways. When creating a new DAG, you probably want to set
 a global <code class="docutils literal"><span class="pre">start_date</span></code> for your tasks using <code class="docutils literal"><span class="pre">default_args</span></code>. The first
@@ -205,7 +220,7 @@ an hour after now as <code class="docutils literal"><span class="pre">now()</spa
 <p>Previously we also recommended using rounded <code class="docutils literal"><span class="pre">start_date</span></code> in relation to your
 <code class="docutils literal"><span class="pre">schedule_interval</span></code>. This meant an <code class="docutils literal"><span class="pre">&#64;hourly</span></code> would be at <code class="docutils literal"><span class="pre">00:00</span></code>
 minutes:seconds, a <code class="docutils literal"><span class="pre">&#64;daily</span></code> job at midnight, a <code class="docutils literal"><span class="pre">&#64;monthly</span></code> job on the
-first of the month. This is no longer required. Airflow will not auto align
+first of the month. This is no longer required. Airflow will now auto align
 the <code class="docutils literal"><span class="pre">start_date</span></code> and the <code class="docutils literal"><span class="pre">schedule_interval</span></code>, by using the <code class="docutils literal"><span class="pre">start_date</span></code>
 as the moment to start looking.</p>
 <p>You can use any sensor or a <code class="docutils literal"><span class="pre">TimeDeltaSensor</span></code> to delay
@@ -224,6 +239,38 @@ backfill CLI command, get overridden by the backfill&#8217;s command <code class
 This allows for a backfill on tasks that have <code class="docutils literal"><span class="pre">depends_on_past=True</span></code> to
 actually start, if it wasn&#8217;t the case, the backfill just wouldn&#8217;t start.</p>
 </div>
+<div class="section" id="how-can-i-create-dags-dynamically">
+<h2>How can I create DAGs dynamically?<a class="headerlink" href="#how-can-i-create-dags-dynamically" title="Permalink to this headline">�</a></h2>
+<p>Airflow looks in you <code class="docutils literal"><span class="pre">DAGS_FOLDER</span></code> for modules that contain <code class="docutils literal"><span class="pre">DAG</span></code> objects
+in their global namespace, and adds the objects it finds in the
+<code class="docutils literal"><span class="pre">DagBag</span></code>. Knowing this all we need is a way to dynamically assign
+variable in the global namespace, which is easily done in python using the
+<code class="docutils literal"><span class="pre">globals()</span></code> function for the standard library which behaves like a
+simple dictionary.</p>
+<div class="code python highlight-default"><div class="highlight"><pre><span></span><span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">10</span><span class="p">):</span>
+    <span class="n">dag_id</span> <span class="o">=</span> <span class="s1">&#39;foo_</span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">i</span><span class="p">)</span>
+    <span class="nb">globals</span><span class="p">()[</span><span class="n">dag_id</span><span class="p">]</span> <span class="o">=</span> <span class="n">DAG</span><span class="p">(</span><span class="n">dag_id</span><span class="p">)</span>
+    <span class="c1"># or better, call a function that returns a DAG object!</span>
+</pre></div>
+</div>
+</div>
+<div class="section" id="what-are-all-the-airflow-run-commands-in-my-process-list">
+<h2>What are all the <code class="docutils literal"><span class="pre">airflow</span> <span class="pre">run</span></code> commands in my process list?<a class="headerlink" href="#what-are-all-the-airflow-run-commands-in-my-process-list" title="Permalink to this headline">�</a></h2>
+<p>There are many layers of <code class="docutils literal"><span class="pre">airflow</span> <span class="pre">run</span></code> commands, meaning it can call itself.</p>
+<ul class="simple">
+<li>Basic <code class="docutils literal"><span class="pre">airflow</span> <span class="pre">run</span></code>: fires up an executor, and tell it to run an
+<code class="docutils literal"><span class="pre">airflow</span> <span class="pre">run</span> <span class="pre">--local</span></code> command. if using Celery, this means it puts a
+command in the queue for it to run remote, on the worker. If using
+LocalExecutor, that translates into running it in a subprocess pool.</li>
+<li>Local <code class="docutils literal"><span class="pre">airflow</span> <span class="pre">run</span> <span class="pre">--local</span></code>: starts an <code class="docutils literal"><span class="pre">airflow</span> <span class="pre">run</span> <span class="pre">--raw</span></code>
+command (described bellow) as a subprocess and is in charge of
+emitting heartbeats, listening for external kill signals
+and ensures some cleanup takes place if the subprocess fails</li>
+<li>Raw <code class="docutils literal"><span class="pre">airflow</span> <span class="pre">run</span> <span class="pre">--raw</span></code> runs the actual operator&#8217;s execute method and
+performs the actual work</li>
+</ul>
+</div>
+</div>
 
 
            </div>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/genindex.html
----------------------------------------------------------------------
diff --git a/genindex.html b/genindex.html
index 20b510f..d4a80fb 100644
--- a/genindex.html
+++ b/genindex.html
@@ -316,6 +316,12 @@
   <dt><a href="code.html#airflow.models.BaseOperator.clear">clear() (airflow.models.BaseOperator method)</a>
   </dt>
 
+      <dd><dl>
+        
+  <dt><a href="code.html#airflow.models.DAG.clear">(airflow.models.DAG method)</a>
+  </dt>
+
+      </dl></dd>
       
   <dt><a href="code.html#airflow.models.TaskInstance.clear_xcom_data">clear_xcom_data() (airflow.models.TaskInstance method)</a>
   </dt>
@@ -409,6 +415,14 @@
   </dt>
 
       
+  <dt><a href="code.html#airflow.models.DAG.deactivate_stale_dags">deactivate_stale_dags() (airflow.models.DAG static method)</a>
+  </dt>
+
+      
+  <dt><a href="code.html#airflow.models.DAG.deactivate_unknown_dags">deactivate_unknown_dags() (airflow.models.DAG static method)</a>
+  </dt>
+
+      
   <dt><a href="code.html#airflow.contrib.hooks.FTPHook.delete_directory">delete_directory() (airflow.contrib.hooks.FTPHook method)</a>
   </dt>
 
@@ -416,12 +430,12 @@
   <dt><a href="code.html#airflow.contrib.hooks.FTPHook.delete_file">delete_file() (airflow.contrib.hooks.FTPHook method)</a>
   </dt>
 
+  </dl></td>
+  <td style="width: 33%" valign="top"><dl>
       
   <dt><a href="code.html#airflow.contrib.hooks.FTPHook.describe_directory">describe_directory() (airflow.contrib.hooks.FTPHook method)</a>
   </dt>
 
-  </dl></td>
-  <td style="width: 33%" valign="top"><dl>
       
   <dt><a href="code.html#airflow.models.BaseOperator.detect_downstream_cycle">detect_downstream_cycle() (airflow.models.BaseOperator method)</a>
   </dt>
@@ -523,6 +537,10 @@
 <table style="width: 100%" class="indextable genindextable"><tr>
   <td style="width: 33%" valign="top"><dl>
       
+  <dt><a href="code.html#airflow.models.TaskInstance.generate_command">generate_command() (airflow.models.TaskInstance static method)</a>
+  </dt>
+
+      
   <dt><a href="code.html#airflow.operators.GenericTransfer">GenericTransfer (class in airflow.operators)</a>
   </dt>
 
@@ -919,6 +937,16 @@
 <table style="width: 100%" class="indextable genindextable"><tr>
   <td style="width: 33%" valign="top"><dl>
       
+  <dt><a href="code.html#airflow.operators.NamedHivePartitionSensor">NamedHivePartitionSensor (class in airflow.operators)</a>
+  </dt>
+
+      
+  <dt><a href="code.html#airflow.models.TaskInstance.next_retry_datetime">next_retry_datetime() (airflow.models.TaskInstance method)</a>
+  </dt>
+
+  </dl></td>
+  <td style="width: 33%" valign="top"><dl>
+      
   <dt><a href="code.html#airflow.models.DAG.normalize_schedule">normalize_schedule() (airflow.models.DAG method)</a>
   </dt>
 
@@ -1021,16 +1049,20 @@
   </dt>
 
       
-  <dt><a href="code.html#airflow.models.BaseOperator.render_template">render_template() (airflow.models.BaseOperator method)</a>
+  <dt><a href="code.html#airflow.contrib.hooks.FTPHook.rename">rename() (airflow.contrib.hooks.FTPHook method)</a>
   </dt>
 
       
-  <dt><a href="code.html#airflow.models.BaseOperator.render_template_from_field">render_template_from_field() (airflow.models.BaseOperator method)</a>
+  <dt><a href="code.html#airflow.models.BaseOperator.render_template">render_template() (airflow.models.BaseOperator method)</a>
   </dt>
 
   </dl></td>
   <td style="width: 33%" valign="top"><dl>
       
+  <dt><a href="code.html#airflow.models.BaseOperator.render_template_from_field">render_template_from_field() (airflow.models.BaseOperator method)</a>
+  </dt>
+
+      
   <dt><a href="code.html#airflow.contrib.hooks.FTPHook.retrieve_file">retrieve_file() (airflow.contrib.hooks.FTPHook method)</a>
   </dt>
 
@@ -1114,12 +1146,12 @@
   <dt><a href="code.html#airflow.operators.SimpleHttpOperator">SimpleHttpOperator (class in airflow.operators)</a>
   </dt>
 
-  </dl></td>
-  <td style="width: 33%" valign="top"><dl>
       
   <dt><a href="code.html#airflow.models.DagBag.size">size() (airflow.models.DagBag method)</a>
   </dt>
 
+  </dl></td>
+  <td style="width: 33%" valign="top"><dl>
       
   <dt><a href="code.html#airflow.operators.SlackAPIOperator">SlackAPIOperator (class in airflow.operators)</a>
   </dt>
@@ -1156,6 +1188,10 @@
   <dt><a href="code.html#airflow.models.DAG.subdags">subdags (airflow.models.DAG attribute)</a>
   </dt>
 
+      
+  <dt><a href="code.html#airflow.models.DAG.sync_to_db">sync_to_db() (airflow.models.DAG static method)</a>
+  </dt>
+
   </dl></td>
 </tr></table>
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/index.html
----------------------------------------------------------------------
diff --git a/index.html b/index.html
index 5ac4744..5e8642c 100644
--- a/index.html
+++ b/index.html
@@ -329,7 +329,15 @@ unit of work and continuity.</p>
 </li>
 </ul>
 </li>
-<li class="toctree-l1"><a class="reference internal" href="faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="faq.html">FAQ</a><ul>
+<li class="toctree-l2"><a class="reference internal" href="faq.html#why-isn-t-my-task-getting-scheduled">Why isn&#8217;t my task getting scheduled?</a></li>
+<li class="toctree-l2"><a class="reference internal" href="faq.html#how-do-i-trigger-tasks-based-on-another-task-s-failure">How do I trigger tasks based on another task&#8217;s failure?</a></li>
+<li class="toctree-l2"><a class="reference internal" href="faq.html#why-are-connection-passwords-still-not-encrypted-in-the-metadata-db-after-i-installed-airflow-crypto">Why are connection passwords still not encrypted in the metadata db after I installed airflow[crypto]?</a></li>
+<li class="toctree-l2"><a class="reference internal" href="faq.html#what-s-the-deal-with-start-date">What&#8217;s the deal with <code class="docutils literal"><span class="pre">start_date</span></code>?</a></li>
+<li class="toctree-l2"><a class="reference internal" href="faq.html#how-can-i-create-dags-dynamically">How can I create DAGs dynamically?</a></li>
+<li class="toctree-l2"><a class="reference internal" href="faq.html#what-are-all-the-airflow-run-commands-in-my-process-list">What are all the <code class="docutils literal"><span class="pre">airflow</span> <span class="pre">run</span></code> commands in my process list?</a></li>
+</ul>
+</li>
 <li class="toctree-l1"><a class="reference internal" href="code.html">API Reference</a><ul>
 <li class="toctree-l2"><a class="reference internal" href="code.html#operators">Operators</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="code.html#baseoperator">BaseOperator</a></li>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/4af0850c/objects.inv
----------------------------------------------------------------------
diff --git a/objects.inv b/objects.inv
index 56e566e..7823a60 100644
Binary files a/objects.inv and b/objects.inv differ