[1/5] incubator-airflow-site git commit: Updating the disclaimer

[ma...@apache.org]

Repository: incubator-airflow-site
Updated Branches:
  refs/heads/asf-site 502934ca8 -> 457b698d9


http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/searchindex.js
----------------------------------------------------------------------
diff --git a/searchindex.js b/searchindex.js
index 901905a..97adc3c 100644
--- a/searchindex.js
+++ b/searchindex.js
@@ -1 +1 @@
-Search.setIndex({envversion:47,filenames:["cli","code","concepts","configuration","faq","index","installation","license","plugins","profiling","project","scheduler","security","start","tutorial","ui"],objects:{"airflow.contrib":{hooks:[1,1,0,"-"],operators:[1,1,0,"-"]},"airflow.contrib.executors.mesos_executor":{MesosExecutor:[1,0,1,""]},"airflow.contrib.hooks":{BigQueryHook:[1,0,1,""],CloudantHook:[1,0,1,""],FTPHook:[1,0,1,""],GoogleCloudStorageHook:[1,0,1,""],SSHHook:[1,0,1,""],VerticaHook:[1,0,1,""]},"airflow.contrib.hooks.BigQueryHook":{get_conn:[1,2,1,""],get_pandas_df:[1,2,1,""],get_service:[1,2,1,""],insert_rows:[1,2,1,""]},"airflow.contrib.hooks.CloudantHook":{db:[1,2,1,""]},"airflow.contrib.hooks.FTPHook":{close_conn:[1,2,1,""],create_directory:[1,2,1,""],delete_directory:[1,2,1,""],delete_file:[1,2,1,""],describe_directory:[1,2,1,""],get_conn:[1,2,1,""],list_directory:[1,2,1,""],retrieve_file:[1,2,1,""],store_file:[1,2,1,""]},"airflow.contrib.hooks.GoogleCloudStorageHook":
 {download:[1,2,1,""],get_conn:[1,2,1,""],upload:[1,2,1,""]},"airflow.contrib.hooks.SSHHook":{Popen:[1,2,1,""],check_output:[1,2,1,""],tunnel:[1,2,1,""]},"airflow.contrib.hooks.VerticaHook":{get_conn:[1,2,1,""]},"airflow.contrib.hooks.gcs_hook":{GoogleCloudStorageHook:[1,0,1,""]},"airflow.contrib.operators":{QuboleOperator:[1,0,1,""],SSHExecuteOperator:[1,0,1,""],VerticaOperator:[1,0,1,""],VerticaToHiveTransfer:[1,0,1,""]},"airflow.contrib.operators.bigquery_operator":{BigQueryOperator:[1,0,1,""]},"airflow.contrib.operators.bigquery_to_gcs":{BigQueryToCloudStorageOperator:[1,0,1,""]},"airflow.contrib.operators.gcs_download_operator":{GoogleCloudStorageDownloadOperator:[1,0,1,""]},"airflow.contrib.operators.hipchat_operator":{HipChatAPIOperator:[1,0,1,""],HipChatAPISendRoomNotificationOperator:[1,0,1,""]},"airflow.executors":{CeleryExecutor:[1,0,1,""],LocalExecutor:[1,0,1,""],SequentialExecutor:[1,0,1,""]},"airflow.hooks":{DbApiHook:[1,0,1,""],DruidHook:[1,0,1,""],HiveCliHook:[1,0,1,"
 "],HiveMetastoreHook:[1,0,1,""],HiveServer2Hook:[1,0,1,""],HttpHook:[1,0,1,""],MsSqlHook:[1,0,1,""],MySqlHook:[1,0,1,""],PostgresHook:[1,0,1,""],PrestoHook:[1,0,1,""],S3Hook:[1,0,1,""],SqliteHook:[1,0,1,""],WebHDFSHook:[1,0,1,""]},"airflow.hooks.DbApiHook":{bulk_dump:[1,2,1,""],bulk_load:[1,2,1,""],get_conn:[1,2,1,""],get_cursor:[1,2,1,""],get_first:[1,2,1,""],get_pandas_df:[1,2,1,""],get_records:[1,2,1,""],insert_rows:[1,2,1,""],run:[1,2,1,""]},"airflow.hooks.DruidHook":{construct_ingest_query:[1,2,1,""],get_conn:[1,2,1,""],load_from_hdfs:[1,2,1,""]},"airflow.hooks.HiveCliHook":{load_file:[1,2,1,""],run_cli:[1,2,1,""],test_hql:[1,2,1,""]},"airflow.hooks.HiveMetastoreHook":{check_for_partition:[1,2,1,""],get_databases:[1,2,1,""],get_metastore_client:[1,2,1,""],get_partitions:[1,2,1,""],get_table:[1,2,1,""],get_tables:[1,2,1,""],max_partition:[1,2,1,""],table_exists:[1,2,1,""]},"airflow.hooks.HiveServer2Hook":{get_pandas_df:[1,2,1,""],get_records:[1,2,1,""]},"airflow.hooks.HttpHook":
 {get_conn:[1,2,1,""],run:[1,2,1,""],run_and_check:[1,2,1,""]},"airflow.hooks.MsSqlHook":{get_conn:[1,2,1,""]},"airflow.hooks.MySqlHook":{bulk_load:[1,2,1,""],get_conn:[1,2,1,""]},"airflow.hooks.PrestoHook":{get_conn:[1,2,1,""],get_first:[1,2,1,""],get_pandas_df:[1,2,1,""],get_records:[1,2,1,""],run:[1,2,1,""]},"airflow.hooks.S3Hook":{check_for_bucket:[1,2,1,""],check_for_key:[1,2,1,""],check_for_prefix:[1,2,1,""],check_for_wildcard_key:[1,2,1,""],get_bucket:[1,2,1,""],get_conn:[1,2,1,""],get_key:[1,2,1,""],get_wildcard_key:[1,2,1,""],list_keys:[1,2,1,""],list_prefixes:[1,2,1,""],load_file:[1,2,1,""],load_string:[1,2,1,""]},"airflow.hooks.SqliteHook":{get_conn:[1,2,1,""]},"airflow.hooks.WebHDFSHook":{check_for_path:[1,2,1,""],get_conn:[1,2,1,""],load_file:[1,2,1,""]},"airflow.macros":{ds_add:[1,3,1,""],ds_format:[1,3,1,""],hive:[1,1,0,"-"],integrate_plugins:[1,3,1,""],random:[1,3,1,""]},"airflow.macros.hive":{closest_ds_partition:[1,3,1,""],max_partition:[1,3,1,""]},"airflow.models":
 {BaseOperator:[1,0,1,""],Connection:[1,0,1,""],DAG:[1,0,1,""],DagBag:[1,0,1,""],TaskInstance:[1,0,1,""]},"airflow.models.BaseOperator":{clear:[1,2,1,""],dag:[1,4,1,""],detect_downstream_cycle:[1,2,1,""],downstream_list:[1,4,1,""],execute:[1,2,1,""],get_direct_relatives:[1,2,1,""],get_flat_relatives:[1,2,1,""],get_task_instances:[1,2,1,""],has_dag:[1,2,1,""],on_kill:[1,2,1,""],post_execute:[1,2,1,""],pre_execute:[1,2,1,""],prepare_template:[1,2,1,""],render_template:[1,2,1,""],render_template_from_field:[1,2,1,""],run:[1,2,1,""],schedule_interval:[1,4,1,""],set_downstream:[1,2,1,""],set_upstream:[1,2,1,""],upstream_list:[1,4,1,""],xcom_pull:[1,2,1,""],xcom_push:[1,2,1,""]},"airflow.models.Connection":{extra_dejson:[1,4,1,""]},"airflow.models.DAG":{add_task:[1,2,1,""],add_tasks:[1,2,1,""],cli:[1,2,1,""],concurrency_reached:[1,4,1,""],crawl_for_tasks:[1,2,1,""],create_dagrun:[1,2,1,""],filepath:[1,4,1,""],folder:[1,4,1,""],get_template_env:[1,2,1,""],is_paused:[1,4,1,""],latest_executi
 on_date:[1,4,1,""],run:[1,2,1,""],set_dependency:[1,2,1,""],sub_dag:[1,2,1,""],subdags:[1,4,1,""],tree_view:[1,2,1,""]},"airflow.models.DagBag":{bag_dag:[1,2,1,""],collect_dags:[1,2,1,""],dagbag_report:[1,2,1,""],get_dag:[1,2,1,""],kill_zombies:[1,2,1,""],process_file:[1,2,1,""],size:[1,2,1,""]},"airflow.models.TaskInstance":{are_dependencies_met:[1,2,1,""],are_dependents_done:[1,2,1,""],clear_xcom_data:[1,2,1,""],command:[1,2,1,""],current_state:[1,2,1,""],error:[1,2,1,""],evaluate_trigger_rule:[1,2,1,""],is_premature:[1,2,1,""],is_queueable:[1,2,1,""],is_runnable:[1,2,1,""],key:[1,4,1,""],pool_full:[1,2,1,""],ready_for_retry:[1,2,1,""],refresh_from_db:[1,2,1,""],run:[1,2,1,""],xcom_pull:[1,2,1,""],xcom_push:[1,2,1,""]},"airflow.operators":{BashOperator:[1,0,1,""],BranchPythonOperator:[1,0,1,""],DummyOperator:[1,0,1,""],EmailOperator:[1,0,1,""],ExternalTaskSensor:[1,0,1,""],GenericTransfer:[1,0,1,""],HdfsSensor:[1,0,1,""],Hive2SambaOperator:[1,0,1,""],HiveOperator:[1,0,1,""],HivePa
 rtitionSensor:[1,0,1,""],HiveToDruidTransfer:[1,0,1,""],HiveToMySqlTransfer:[1,0,1,""],HttpSensor:[1,0,1,""],MetastorePartitionSensor:[1,0,1,""],MsSqlOperator:[1,0,1,""],MsSqlToHiveTransfer:[1,0,1,""],MySqlOperator:[1,0,1,""],MySqlToHiveTransfer:[1,0,1,""],PostgresOperator:[1,0,1,""],PrestoCheckOperator:[1,0,1,""],PrestoIntervalCheckOperator:[1,0,1,""],PrestoValueCheckOperator:[1,0,1,""],PythonOperator:[1,0,1,""],S3KeySensor:[1,0,1,""],S3ToHiveTransfer:[1,0,1,""],ShortCircuitOperator:[1,0,1,""],SimpleHttpOperator:[1,0,1,""],SlackAPIOperator:[1,0,1,""],SlackAPIPostOperator:[1,0,1,""],SqlSensor:[1,0,1,""],TimeSensor:[1,0,1,""],TriggerDagRunOperator:[1,0,1,""],WebHdfsSensor:[1,0,1,""]},"airflow.operators.BashOperator":{execute:[1,2,1,""]},"airflow.operators.SlackAPIOperator":{construct_api_call_params:[1,2,1,""],execute:[1,2,1,""]},"airflow.operators.docker_operator":{DockerOperator:[1,0,1,""]},"airflow.operators.sensors":{BaseSensorOperator:[1,0,1,""]},airflow:{executors:[1,1,0,"-"],h
 ooks:[1,1,0,"-"],macros:[1,1,0,"-"],models:[1,1,0,"-"],operators:[1,1,0,"-"]}},objnames:{"0":["py","class","Python class"],"1":["py","module","Python module"],"2":["py","method","Python method"],"3":["py","function","Python function"],"4":["py","attribute","Python attribute"]},objtypes:{"0":"py:class","1":"py:module","2":"py:method","3":"py:function","4":"py:attribute"},terms:{"00am":1,"01t23":11,"10gb":1,"10pm":2,"128m":1,"2am":1,"5gb":1,"__class__":2,"__doc__":2,"__init__":[1,2],"__name__":[2,8],"_cmd":3,"_host":12,"abstract":[1,2,5,8],"boolean":[1,2],"byte":[1,4],"case":[1,2,4,11,12,14],"char":1,"class":[0,1,2,6,7,8,14],"default":0,"export":[1,13],"final":1,"float":1,"function":1,"import":[1,2,4,5,6,7,8,11,12],"int":1,"long":1,"new":[1,2,4,8,11,12],"return":[0,1,2,8],"short":1,"static":[1,5,8],"super":[1,12],"switch":[3,12],"true":[1,2,4,12,14],"try":[1,2,3],"var":1,"while":[1,2,4,5,7,11,13,14,15],abbrevi:2,abil:2,abl:[3,4,12,13,14],about:[1,2,3,7,10,11,14],abov:[2,3,7,12,15],abs
 enc:2,absolut:2,accept:[1,5,7],access:[0,1,2,3,8,12],access_logfil:0,access_token:15,accord:12,accordingli:2,account:[1,12],acquir:1,across:[3,15],act:[1,2,7,14],action:[1,15],activ:[1,2,4,8,11,12,14],actual:[1,2,4,14],acycl:[1,2,5],add:[1,2,3,7,12],add_task:1,addendum:7,addit:1,addition:12,addprinc:12,adhoc:[0,1,5],adjust:[3,12],admin:[2,3,4,8,9,12,13],admin_view:8,advanc:[1,14],advantag:12,advis:[1,7],affect:2,after:[1,2,4,11],afterward:1,against:[1,2,4,7,12,14],agari:10,aggregate_db_message_job:2,aggregate_db_message_job_cmd:2,ago:4,agre:7,agreement:[2,7],airbnb:[1,2,10,14],airflow:[0,1,2,3,4],airflow__:3,airflow__core__sql_alchemy_conn:3,airflow_config:3,airflow_conn_:[2,3],airflow_conn_postgres_mast:2,airflow_hom:[3,4,8,13],airflow_set:2,airflow_tmp_dir:1,airflowignor:1,airflowplugin:8,airflowtestplugin:8,aiyzngq:1,alchemi:3,alert:[1,2,8],all:[0,1,2,3,4,5,6,7,11,12,14],all_db:6,all_don:[1,2],all_fail:[1,2],all_success:[1,2],alleg:7,alloc:1,allow:[0,1,2,3,4,5,8,9,11,12,13,14,15]
 ,allow_large_result:1,allowed_st:1,allowed_team:12,allowinsert:1,alon:7,along:[1,4,7,14],alongsid:[1,7,11],alreadi:1,alright:14,also:[1,2,3,4,6,7,8,11,12,14],alter:[1,2,3,4,12],altern:[1,11],altert:1,although:12,alwai:[1,2,15],amazon:3,amount:[1,2],analys:15,analyz:2,anand:10,anchor:1,ani:[1,2,3,4,7,14,15],annot:7,announc:10,anomali:8,anoth:[1,2,4,10],ansibl:3,anyon:12,anyth:[2,14],anytim:2,anywher:[1,2],aoen:10,apach:1,api:0,api_call_param:1,api_kei:15,api_param:1,api_vers:1,apikei:15,apiv2:1,app:1,app_id:1,appar:1,appear:[1,2,7],append:3,appendix:7,appli:[1,2,6,7,11],applic:[1,7,8,9,12],appropri:[2,7,12],arbitrari:[2,5],architectur:5,archiv:[1,7],archive1:1,archive2:1,are_dependencies_met:1,are_dependents_don:1,aren:[1,2],arg:1,argument:[0,1],aris:7,around:[1,5,14],arrai:[1,5,8],arthur:10,artwr:10,ascii:1,asf:5,assert:7,asset:1,assign:1,associ:[1,2,7,11],assum:[1,2,3,7,14],asterisk:12,async:6,atom:2,attach:[1,2,7],attach_to:1,attack:12,attempt:[0,1],attent:[1,4],attr:1,attribut:[0
 ,1,2,7],audit:8,auth:[1,12],auth_backend:12,auth_token:1,authen:12,authent:[1,2,5,6],authmechan:1,author:[1,3,5,7,12,14,15],authorship:7,autocommit:1,automat:[1,2,3],avail:[1,2,3,7,8,9],averag:1,avoid:2,awai:1,awar:2,azkaban:5,back:[1,4,8],backend:1,backfil:[0,1,4,5,11,13],background:[1,14],backup:3,bag:1,bag_dag:1,balanc:2,bar:[1,2],bare:14,base64:4,base:[1,2,3,4,7,8,11],base_executor:[1,8],base_hook:[1,8],base_url:1,basedn:12,baseexecutor:[1,8],basehook:[1,2,8],baseview:8,bash:[1,2,14],bash_1:2,bash_command:[1,2,14],bash_command_to_run:3,bash_queu:14,bashoper:[1,2,14],basi:7,basic:[2,6,8,9,14],bcrypt:12,beauchemin:10,becaus:[1,2],becom:[1,2,5,8,14],beelin:1,been:[1,2,3,7,11,12],befor:[0,1,2,4,12],begin:[2,11],behalf:7,behav:1,behavior:[1,2],behind:[1,3,6,11],belong:12,below:[1,7,8,13,14],benefici:7,best:[1,10],better:14,between:[1,2,14],beyond:[1,4],bigqueri:1,bigquery_conn_id:1,bigquery_default:1,bigquery_oper:1,bigquery_to_gc:1,bigqueryhook:1,bigqueryoper:1,bigquerytocloudstorag
 eoper:1,bin:2,bind:7,bind_password:12,bind_us:12,bit:3,blob:[1,14],block:[1,2,14,15],blog:[7,10],blueprint:8,bodi:1,boilerpl:7,bolk:10,bolkedebruin:10,bonu:1,bool:1,boot:3,both:[1,4],bother:14,boto:1,bottleneck:15,boundary_queri:1,box:[1,3,13],bql:1,bracket:[7,14],branch:1,branch_fals:2,branchpythonoper:[1,2],broker:[0,3],broker_api:0,broker_url:3,brought:10,brows:[2,4],bruin:10,bucket:[1,3],bucket_kei:1,bucket_nam:1,buffer:1,bug:3,build:[1,2,3,5],built:[1,2,3,5,8,9,14],bulk:[4,15],bulk_dump:1,bulk_load:1,bump:2,bundl:1,burn:0,but_this_dag_will_not:2,c12318391:1,cacert:12,cach:12,call:[1,2,3,14],callabl:1,callback:12,can:[1,2,3,4,5,6,8,9,11,12,13,14,15],cannot:[1,2,7,14],capabl:8,capac:2,card:1,care:[0,3],carri:[2,7],cast:1,cat:1,categori:8,caus:[4,7],celeri:[0,1,2],celery_result_backend:3,celeryexecutor:[1,2,3,4,6,11],cell:1,central:2,cert:1,certain:[1,2],certif:1,cfg:[1,2,3,4,11,12,13,14],chain:2,chang:[3,5,7,11,12,13,14],channel:[1,10],charact:7,character:2,charg:7,charset:1,char
 t:[5,8],chat:10,check:[0,1,2,4],check_for_bucket:1,check_for_kei:1,check_for_partit:1,check_for_path:1,check_for_prefix:1,check_for_wildcard_kei:1,check_head:1,check_oper:1,check_output:1,checkoper:1,chef:[3,10],child:[1,2],child_dag:2,child_dag_nam:2,chmod:12,choic:[0,1,14],choos:[1,2,7],chri:[5,10],circuit:1,claim:7,clariti:5,claus:1,clean:1,cleanup:1,clear:[0,1,2,4,11,15],clear_xcom_data:1,cli:[1,2,3,4,11],click:[4,12,14,15],client:[1,6,12],client_id:12,client_secret:12,close:[1,4,12],close_conn:1,closest:1,closest_ds_partit:1,cloud:[1,3,6],cloudant:[1,6],cloudant_conn_id:1,cloudant_default:1,cloudanthook:1,cluster:1,cluster_label:1,cmd:1,cmdline:1,code:[0,1,2,3,5,7,8,14],col:1,collabor:5,collect:[1,2,8],collect_dag:1,color:1,column:1,com:[1,2,12,14],combin:[2,7],come:[1,11],comma:[0,1,2],command_typ:1,comment:7,commerci:7,commit:[1,10,12],commit_everi:1,committ:5,common:[2,3,4,7,10,14],commonli:1,compani:8,compar:5,compil:[4,7],complet:[1,2,3,5],complex:[1,2,5,8],compli:7,compli
 anc:7,complic:14,compon:8,compos:[2,12,14],comprehens:15,compress:1,comput:[1,7],conceptu:2,concern:2,concur:4,concurr:[0,1,4],concurrency_reach:1,condit:[1,2,6,7],conf:[0,1,3],config:[8,12],configur:[1,2],confirm:[0,4,11],confus:[1,2,4],conjunct:[0,2,13],conn_id:[1,2,3],conn_typ:1,connect:1,connect_timeout:1,connecton:9,consequenti:7,consid:[1,2,3],consider:1,consist:[1,2,5,7],conspicu:7,constant:[1,2],constitu:2,constitut:7,constraint:1,constru:7,construct:1,construct_api_call_param:1,construct_ingest_queri:1,constructor:[1,14],contact:[3,7],contain:[1,2,3,7,11,12,14,15],content:[1,2],context:1,continu:[1,5],contract:7,contrib:[1,2,12],contributor:[7,10],contributori:7,control:[1,2,7,15],conveni:[1,2],convent:2,convers:7,cool:1,coordin:1,copi:[1,3,7,12],copyright:[7,12],core:1,correct:2,correpsond:2,correspond:[1,2,3,4],could:[1,2,14],count:1,counterclaim:7,cours:12,cover:11,cpu:[1,8],craft:1,crawl:[1,11],crawl_for_task:1,creat:[1,2,3,4,8,9,11,12,13,14,15],create_dagrun:1,create_d
 irectori:1,creation:1,credenti:12,credit:12,criccomini:10,criteria:[1,2],criterion:1,critic:[1,2],cron:[1,4,11],cross:[2,7,14],crt:12,crypto:[3,4,6],csv:1,curli:14,current:[1,15],current_st:1,cursor:1,custom:8,customari:7,cycl:[1,2,14],daemon:[0,1,3],dag:[0,1],dag_1:2,dag_2:2,dag_fold:[1,2],dag_id:[0,1,2,14],dag_md:2,dag_run:1,dag_run_obj:1,dag_stat:0,dagbag:[0,1],dagbag_report:1,dagfil:2,dagrun:[0,1,4],dagrun_timeout:1,dags_fold:3,dai:[1,11,13,14],daili:[1,4,11],damag:7,dan:[1,10],data:[1,2,5,6,8],data_profiler_filt:12,databas:[0,1,2,3,5,6,8,9,13,14],datafram:1,dataset:1,datasourc:1,date:[0,1,2,7,11,14],date_filter_column:1,datestamp:1,datetim:[1,2,4,11,14],dateutil:1,davydov:10,days_back:1,db_tabl:1,db_tap_id:1,db_update_kei:1,db_update_mod:1,dbapi_hook:1,dbapihook:1,dbexportcmd:1,dbimportcmd:1,dbtap_id:1,dbtapquerycmd:1,deal:4,debug:[0,1,14],decid:[1,3],decis:5,declar:1,deep:1,def:[1,2,8],default_arg:[1,2,4,14],default_login:12,default_queu:2,defend:7,defer:2,deferred_op:2,defin:
 [1,2,3,4,5,7,8,11,12,14],definit:[2,5,7,8],del:1,delai:[1,4],delegate_to:1,delet:[1,2,11,15],delete_directori:1,delete_fil:1,deliber:[1,7],delimit:[0,1,2],demonstr:2,depend:[0,1,2,3,4,5,6,11],depends_on_past:[0,1,2,4,14],deploi:8,deploy:8,depth:14,deriv:[1,3,7,8],descend:2,describ:[1,2,7],describe_directori:1,descript:[1,2,7],deseri:[0,1,2],deserialize_json:2,design:[2,7,11],desir:[1,14],destin:1,destination_cloud_storage_uri:1,destination_conn_id:1,destination_dataset_t:1,destination_filepath:1,destination_t:1,detail:[1,2,4,12,15],detect:[1,8],detect_downstream_cycl:1,determin:[1,2,7],dev:[3,6,10],devel:6,devel_hadoop:6,develop:[0,1,12,14],deviat:1,dialog:11,dict:[0,1,2],dictionari:[1,2,14],did:14,die:2,differ:[1,2,7,8,14,15],dimens:1,direct:[1,2,3,5,7],directli:[1,2,4,9],directori:[0,1,2,3,12,14],dirti:13,disabl:1,disclaim:7,discuss:7,disk:1,displai:[3,7,11,14],disregard:[1,14],distinct:1,distribut:[1,2,6,7],do_pickl:0,doc:[1,2,4,14],doc_json:2,doc_md:2,doc_rst:2,doc_yaml:2,docker
 :[1,10],docker_oper:1,docker_url:1,dockeroper:[1,2],document:1,doe:[0,1,2,3,4,5,6,7,12,14],does_not_exist:1,doesn:[1,2,14],domain:12,don:[1,2,3,6,7,11],done:[1,2,11,12,14],donot_pickl:[0,1],door:2,dot:[1,2],doubl:[1,3,14],down:[0,2],download:[1,3],downstream:[0,1,2,11],downstream_list:1,downstream_task_id:1,drive:3,driven:[1,8],driver:3,drop:[1,8],druid:[1,6],druid_datasourc:1,druid_ingest_conn_id:1,druid_ingest_default:1,druid_query_conn_id:1,druid_query_default:1,druidhook:1,dry:0,dry_run:0,ds_add:[1,14],ds_format:1,ds_nodash:1,dubiou:1,dummi:[1,2,8],dummy_1:2,dummy_oper:2,dummy_task:2,dummyoper:[1,2],dump:1,durat:5,dure:[3,15],dynam:[1,2,4,5],each:[1,2,4,7,10,11,14],earlier:14,easi:[2,3,5,8,9,12,15],easier:[1,7],easiest:6,easili:[1,2,5,9,14],echo:[2,14],ecosystem:8,edit:[3,15],editori:7,effect:[1,2],effici:1,effort:5,egg:3,either:[1,2,3,7,12,13],elabor:7,elaps:1,elast:1,electron:7,eleg:5,element:3,els:13,elsewher:2,email:[1,2,10,12,14],email_on_failur:[1,14],email_on_retri:[1,14]
 ,emailoper:[1,2],emit:[2,14],empti:[1,2],enabl:[1,2,3,5,6],enclos:7,encod:[1,4],encrypt:[1,3,4,6,12],encrypt_s3_log:3,end:[1,7,11],end_dat:[0,1,14],endors:5,endpoint:1,enforc:[2,4],engin:[1,4,5],enterpris:5,entir:[0,4],entiti:7,entri:[1,11],env:[1,2],environ:[1,2,3,5,6,9,11,14,15],ep_data_pipeline_db_msg_agg:2,equival:[2,6,14],era:4,error:[0,1],error_logfil:0,especi:[2,4,14],essenti:1,etc:[1,2,3,12],eval:1,evalu:[1,2,4,11,14],evaluate_trigger_rul:1,even:[1,2,7,9],event:[2,7,14],eventlet:0,ever:1,everi:[1,2,11,12,14],everyon:14,everyth:[14,15],exactli:[1,15],exampl:[1,2,3,5,7],example1:13,example_bash_oper:13,example_dag:[2,14],example_team_1:12,example_team_2:12,exce:1,except:[1,2,4,7,14],exchang:[2,5],exclud:[0,7],exclude_subdag:0,exclus:[7,11],execut:[0,1,2,3,4,5,7,11,14],execution_d:[0,1,2,14],execution_delta:1,execution_timeout:[1,2],exercis:7,exhaust:3,exist:[0,1,2,3,4,14],exit:[0,12],expect:[1,2,4,5],expir:1,explain:1,explan:14,explicit:5,explicit_op:2,explicitli:[1,2,7,12,14]
 ,export_dir:1,export_format:1,expos:[1,2,3,8,12],express:[1,4,7,11],ext:1,extend:5,extens:[1,5],extern:[1,2,5,8,10],external_dag_id:1,external_task_id:1,external_trigg:1,externaltasksensor:1,extra:[1,2,5],extra_dejson:1,extra_opt:1,extract:1,extract_queri:1,fact:2,factori:2,fail:[0,1,2,3,4,11,14,15],failur:[1,3,4,7],fairli:13,fall:10,fals:[0,1,3,11,14],familiar:[2,14],faq:1,fast:13,fault:[1,2],featur:[1,2,3,6,8,14,15],feb:12,fee:7,fernet:4,fernet_kei:4,fetch:[1,3],few:[1,3,9,13,14],field:[1,7],field_delimit:1,field_dict:1,fields_terminated_bi:1,fifti:7,figur:[1,2,11],file1:1,file2:1,file:[0,1,2,3,5,7,8,12,13],filenam:[1,2],filepath:1,filestatu:1,filesystem:3,fill:[1,2,9,12],filter:[0,1,2,12],filter_by_own:12,find:[1,3,4,14,15],fine:14,finish:1,fire:[1,2,3],first:[0,1,2,3,4,10,11,13,14],fit:[1,5,7],fix:[1,11],flag:[1,11],flag_upstream_fail:1,flask:[0,8,9],flask_admin:8,flask_blueprint:8,flask_login:12,flat:1,flavor:8,flexibl:[1,14],flower:[0,3],folder:[1,3,8,11,13,14],follow:[1,2,3,5
 ,7,12,14],foo:[1,2],foo_dag_id:4,foobar:3,forc:[0,1],force_pul:1,foreground:0,foreign:1,form:[2,5,7],format:[1,2,3,7],forward:[1,4],found:[1,3,12],foundat:[5,7,10,13],four:2,framework:[3,8],free:[2,7],frm:1,from:[0,1,2,3,4,5,7,8,9,10,11,12,13,14,15],ftp:1,ftp_conn_id:1,ftp_default:1,ftphook:1,full:[1,10],full_filepath:1,fulli:[4,5,12],functiontyp:1,fundament:14,further:[5,13,14],futur:[1,11],galaxi:8,gantt:5,garag:2,gate:12,gcp_api:6,gcp_api_base_hook:1,gcs_download_oper:1,gcs_hook:1,gender:1,gener:[0,1,2,5,7,8,12,14,15],generictransf:1,get:[0,1,2,3,4,5],get_bucket:1,get_conn:1,get_connect:2,get_cursor:1,get_dag:1,get_databas:1,get_direct_rel:1,get_first:1,get_flat_rel:1,get_kei:1,get_metastore_cli:1,get_pandas_df:1,get_partit:1,get_partitions_by_filt:1,get_record:1,get_servic:1,get_tabl:1,get_task_inst:1,get_template_context:1,get_template_env:1,get_wildcard_kei:1,gevent:0,ghe:5,ghe_oauth:12,ghost:1,git:3,github:[1,5,7,10],github_enterpris:12,github_enterprise_auth:12,githubusercon
 t:1,gitignor:1,gitter:10,give:[2,7],given:[1,2],glanc:15,global:[1,2,4],goal:14,goe:[1,4,14],good:2,goodby:2,goodwil:7,googl:[1,3,6],google_cloud_storage_conn_id:1,google_cloud_storage_default:1,googlecloudbasehook:1,googlecloudstoragedownloadoper:1,googlecloudstoragehook:1,govern:7,grab:1,grade:3,grai:1,grant:7,graph:[1,2,5],great:[2,3],greater:1,green:1,grossli:7,group:[1,3,12],gunicorn:[0,6],hack:1,had:[1,4],hadoop:[6,10,12],hadoop_dependency_coordin:1,hadoopcmd:1,hand:[12,13],handi:10,handl:[1,2,3],happen:[2,11,14],hard:[1,2],harmless:7,has_dag:1,hash:[1,12],hasn:[1,2],have:[1,2,3,4,5,6,7,8,9,11,12,14],haven:[1,2,14],hdf:[1,2,6],hdfs_conn_id:1,hdfs_default:1,hdfs_path:1,hdfscli:1,hdfssensor:1,head:[13,14],header:1,heartbeat:[1,2],heavier:1,hello:[2,8],help:[0,7,8,12],here:[1,2,3,4,6,8,10,11,13,14,15],herebi:7,herein:7,hex:1,hidden:15,hierarchi:[1,14],high:1,highchart:9,higher:1,highli:3,highlight:14,hipchat:1,hipchat_oper:1,hipchatapioper:1,hipchatapisendroomnotificationoper:1,h
 istori:5,hit:[1,2],hive2mysqloper:1,hive2sambaoper:[1,6],hive:[1,2,3,6,8,12],hive_cli_conn_id:1,hive_cli_default:1,hive_cli_param:1,hive_conn_id:1,hive_t:1,hiveclihook:1,hivecmd:1,hiveconf:1,hiveconf_jinja_transl:1,hivemetastorehook:1,hiveoper:[1,2,3,12],hivepartitionsensor:[1,2],hivesensoroper:1,hiveserver2:1,hiveserver2_conn_id:1,hiveserver2_default:1,hiveserver2hook:1,hivetodruidtransf:1,hivetomysqltransf:1,hold:7,home:13,homogen:3,hope:11,horribl:14,host:[1,3,12],hostnam:[0,1,2,3],hour:[1,2,4,11],hourli:[1,4,11],hous:2,how:[1,2,3,4,7,8,11,14,15],howev:[2,7,12],hql:1,html:[1,8,14],html_content:1,http:[1,2,7,8,12,14],http_conn_id:1,http_default:1,httphook:1,httpoper:2,httpsensor:1,human:1,icon:1,icon_url:1,idea:1,idempot:1,identif:7,identifi:[0,1,7,14,15],ignor:[0,1],ignore_depend:[0,1],ignore_depends_on_past:[0,1],ignore_first_depends_on_past:[0,1],iii:7,imag:1,immedi:[1,2],imper:3,imperson:12,implement:[1,2],impli:[1,7],improv:7,impyla:1,inabl:7,inact:4,inc:7,incident:7,includ:[
 0,1,2,3,7,11,14],include_adhoc:[0,1],include_downstream:1,include_exampl:1,include_prior_d:1,include_queu:1,include_upstream:1,inclus:7,incorpor:7,incur:7,indemn:7,indemnifi:7,independ:[1,2],index:1,indic:[1,2,5,7],indirect:7,individu:[1,7,11,14],infer:[1,2],inferred_op:2,infil:1,infin:5,info:1,inform:[1,2,3,7,11,12,14],infrastructur:[1,5],infring:7,ingest:1,inherit:[1,14],init:3,initctl:3,initdb:[0,3,13],initi:[0,3,12,13],initialis:1,inject:8,inlin:1,input:1,input_format:1,insecur:12,insecurecli:1,insert:[1,2],insert_row:1,insid:[1,2],inspect:[11,13],instal:[1,2,3,4,5],instanc:[0,1],instanti:[1,2,5,11],instati:14,instead:[0,1,4],insteadnhttp:1,institut:7,instruct:2,insur:1,integr:1,integrate_plugin:1,intention:7,inter:2,interact:[1,2,3,8,9],interest:[1,14],intergr:8,intern:0,interpret:[1,2],interv:1,intervalcheckoper:1,intoth:1,introduc:4,intuit:14,invari:[1,2],invok:1,ipython:2,irrevoc:7,is_paus:1,is_prematur:1,is_queueabl:1,is_runn:1,ish:1,isn:[1,2,4],isoformat:1,issu:[1,2,5,7,10
 ],item:2,iter:1,itself:[1,2,12,14],januari:[7,11],jar:1,java:1,jdbc:[1,6],jeremiah:10,jinja2:1,jinja:1,jinja_env:1,jlowin:10,job:[0,1,2,3,4,11,13,15],job_id:1,jobtrack:1,join:[2,10],json:[0,1,2],jupyt:2,just:[0,1,2,4,8,14],kadmin:12,kapil:3,keep:[1,2,5],kei:[0,1,2,3,4,12,15],kerber:[6,12],kerbero:[0,5,6],kernel:1,key_fil:1,keytab:[0,12],keyword:1,kick:[3,11],kill:[1,2],kill_zombi:1,kind:7,know:[3,14],knowledg:8,known:6,known_host:1,kruyk:10,kwarg:1,kwd:1,l447:1,lab:1,label:[1,9],lai:13,lambda:[1,2],land:[1,8],languag:[1,7,14],larg:1,larger:1,last:1,late:15,later:9,latest:[0,1,6,14],latest_d:1,latest_execution_d:1,latin_stat:1,launch:12,law:7,lawsuit:7,layout:14,ldap3:12,ldap:[5,6],ldap_auth:12,ldap_ca:12,lead:[1,2],lean:5,least:[1,2,7],leav:1,led:2,left:[2,12],legaci:4,legal:7,len:1,leo:10,less:1,lesson:10,let:[0,1,2,3,9,14,15],level:[1,2,5,11,12],leverag:[1,2,14],liabil:7,liabl:7,lib:[1,2,3],libffi:3,librari:[1,2,3,5,8,14],libz:2,licens:5,licensor:7,life:2,light:[1,2],lighter:1,lig
 htweight:2,like:[1,2,3,6,8,12,14],limit:[1,2,3,5,7],link:[2,5,7,8],list:[0,1,2,3,4,6,7,8,10,12,14,15],list_dag:[0,4,14],list_directori:1,list_kei:1,list_prefix:1,list_task:[0,4,14],listen:2,lit:1,liter:1,litig:7,live:1,load:[0,1,2],load_fil:1,load_from_hdf:1,load_str:1,local:[0,1,2,3,12,14],local_executor:1,local_full_path_or_buff:1,local_infil:1,local_port:1,localexecutor:[0,1,2,3,11],localhost:2,locat:[0,1,2,3,12,14],lock:1,lock_for_upd:1,log:[0,1],log_fil:0,logfil:0,loggingmixin:1,logic:[1,2,14],login:[1,2,3,12],logist:8,look:[0,1,2,5,8,10,11,14],loss:7,lot:11,lowin:10,machin:[1,2,3,12],made:[1,2,7],mai:[1,2,3,4,7,11,14],mail:[7,10],main:[1,2,4,8],main_dag:2,maintain:[1,5,10],make:[1,2,3,4,5,7,12,14,15],malfunct:7,malik:3,man:6,manag:1,mani:[0,1,2,4,8,14,15],manner:5,manual:[1,2,3,11],map:1,mapr:1,mark:[0,1,2,7,11],mark_success:[0,1,11],markdown:2,markup:1,master:[1,2,14],match:[0,1,2],matter:1,max:[1,10],max_active_run:[1,4],max_partit:1,maxim:10,maxime_beauchemin:0,maximum:1,ma
 yb:2,mean:[1,2,3,4,7,12,14],meaning:1,meant:1,mechan:[1,7,12],media:7,medium:7,meet:[1,7],mem_limit:1,member:12,memberof:12,memori:1,menu:[2,4,5,8,13],menu_link:8,menulink:8,merchant:7,mere:7,merg:14,meso:1,mesos_executor:1,mesosexecutor:[1,3,11],messag:[1,2,5],message_format:1,met:[1,3,4,11],metadata:[0,1,2,3,4,5,6,8,11],metastor:1,metastore_conn_id:1,metastore_default:1,metastore_mysql:1,metastorepartitionsensor:1,method:[1,2],metric:[1,8],metric_spec:1,metrics_threshold:1,microsoft:[1,6],midnight:[4,11],might:[1,2,4,11,14],migrat:12,mime:1,mime_typ:1,min:4,mind:[2,5],minimum:[1,6,11],minut:[2,4,11,14],mismatch:2,miss:[1,2],mistercrunch:10,mix:14,mkdir:2,mlsd:1,mode:[0,1,14],modif:7,modifi:[1,2,3,7,9],modul:[1,2,5,8,10,12],modular:5,monitor:[2,3,5,8,11,15],monospac:2,month:[4,11],monthli:[4,11],more:[1,2,3,5,7,8,11,12,14,15],morn:11,most:[1,2,12,14,15],mostli:[1,5,11],mount:[1,3],move:[1,2,4,5],msg:1,mssql:[1,6],mssql_conn_id:1,mssql_default:1,mssqlhook:1,mssqloper:1,mssqltohivetr
 ansf:1,much:[1,2,11],multi:5,multipart:1,multipart_byt:1,multipl:[1,2],multipli:1,multiprocess:1,must:[1,2,3,4,7,12,14],mutat:2,my_conn_str:3,my_dag1:2,my_dag2:2,my_dag:2,my_databas:1,my_funct:2,my_param:14,my_tabl:1,my_useful_packag:2,myarg:1,mypackag:12,mys3conn:3,mysql:[1,2,3,6],mysql_conn_id:1,mysql_default:1,mysql_postoper:1,mysql_preoper:1,mysql_tabl:1,mysqldb:1,mysqlhook:1,mysqloper:[1,3],mysqltohivetransf:1,name:[1,2,3,7,8,12],namespac:[1,8],nav:12,navig:12,ndfor:14,necessarili:[5,7],need:[1,2,3,4,5,6,8,11,12,13,14],neg:[1,11],neglig:7,neighbour:1,nest:[1,14],network:[1,12],network_mod:1,never:[0,1,4],new_user_email:12,new_user_nam:12,newli:5,newobject:1,next:[1,2,5,11,12],nhere:1,nice:2,night:2,nlst:1,no_confirm:0,no_host_key_check:1,node:[0,1,2],non:[1,3,7],none:[1,2,8,11],noram:12,norandkei:12,normal:[1,2,7],notat:1,note:1,notebook:2,noth:[1,7],notic:[2,7,14],notif:1,notifi:1,notwithstand:7,nov:10,now:[1,2,3,4,12,14],nuanc:2,num:1,num_run:0,num_shard:1,number:[0,1,2,3,5],
 oauth2:12,oauth_callback_rout:12,oauth_key_from_github_enterpris:12,oauth_secret_from_github_enterpris:12,obj:1,object:[0,1,2,3,4,7,8,11,14],objectclass:12,oblig:7,observ:1,obtain:7,occur:[1,2],occurr:1,octet:1,off:[1,3,11],offer:[7,8],offici:[1,10],offset:1,often:[1,2],on_failure_callback:1,on_kil:1,on_retry_callback:1,on_success_callback:1,onboard:4,onc:[1,2,3,4,11,14],one_fail:[1,2],one_success:[1,2],onli:[0,1,2,3,4,6,7,11,12,13],only_fail:0,only_if_upd:1,only_run:0,oozi:5,op1:2,op2:2,op3:2,op4:2,op_arg:1,op_kwarg:1,open:[1,2,4,10,12],oper:0,oppos:[1,2],optim:1,option:[0,1,2],orchestr:[1,5],order:[1,2,3,11,12],org:[1,7,8,10,14],organ:[2,8],organiz:12,origin:7,orm:1,other:[1,2,3,5,7,9,11,14],otherwis:[1,7,14],our:14,out:[1,2],outag:8,outgrow:13,outlier:15,output:[1,14],output_encod:1,output_format:1,outsid:11,outstand:7,over:[0,1,2,3,13,15],overlap:15,overrid:[0,1,14],overridden:[1,4],overview:15,overwhelm:2,overwrit:1,own:[1,2,3,5,7],owner:[1,2,7,12,14],ownership:7,packag:1,packa
 ge1:2,page:[2,4,7,10,15],pai:[1,4],pair:[1,15],panda:1,parallel:[1,2,3,13],param:[0,1,14],paramet:[0,1,2,3,4,9,14],parameter:[2,5,9],parent:2,parent_dag:[1,2],parent_dag_nam:2,pars:[1,4,8,14],part:[1,2,7,9,12],parti:7,particular:7,partit:[1,2],partition_nam:1,partition_spec:1,partli:4,pass:[1,2,4,8,14],pass_valu:1,passwd:15,password:[1,2,3,4,5,6],password_auth:12,passwordus:12,past:[1,2,4,11,15],patent:7,path2:1,path:[1,2,3],patrick:10,patrickleotardif:10,pattern:[1,2],paus:[0,1],payload:1,pem:1,peopl:[1,8,14],pep:1,percent:7,perfect:2,perform:[0,1,2,3,5,7,15],perhap:[2,15],period:[1,2,4,11,14],permiss:[7,12],perpetu:7,persist:[1,11],perspect:2,pertain:7,phase:8,physic:1,pick:[2,3,9],pickl:[0,1,2,3],picklabl:1,pickle_id:1,pickleabl:1,pid:[0,13],pig:[1,2],pigcmd:1,piggi:8,pigoper:1,pin_100:1,pip:[2,6,13],pipelin:[1,2,3,5,9],pitfal:10,place:[1,2,7,14],placehold:1,plain:[1,3],plan:4,platform:[1,2,5,6],pleas:[2,3,10,12,14],plug:8,plugin:[1,5],pluginexecutor:8,pluginhook:8,pluginoper:8,p
 lugins_manag:8,png:1,point:[1,2,3,4,13,14],poke:1,poke_interv:1,polymorph:1,pool:[0,1],pool_ful:1,popen:1,popul:14,port:[0,1,3,12,13],posit:[0,1],possibl:[0,1,2,3,7,12,13,14,15],post:[1,10],post_execut:1,postgr:[1,2,3,6],postgres_conn_id:1,postgres_default:1,postgres_mast:2,postgreshook:1,postgresoper:1,potenti:2,power:[2,5,7,14],practic:[1,10],pre:4,pre_execut:1,preced:[1,3,14],prefer:[7,11,13],prefix:[1,2,3],preoper:1,prepar:[2,7],prepare_templ:1,prepped_request:1,present:1,preset:11,presto:1,presto_conn_id:1,presto_default:1,prestocheckoper:1,prestocmd:1,prestohook:1,prestointervalcheckoper:1,prestotomysqloper:2,prestovaluecheckoper:1,pretti:14,prevent:[1,2,12],previou:[0,1,2,4,14],previous:2,primari:1,primit:1,princip:[0,2,12],print:[0,1,2,7,14],print_dat:14,print_head:1,prior:1,priorit:2,prioriti:[1,2],priority_weight:[1,2,14],privaci:7,probabl:[3,4,12],problem:2,problemat:2,proce:[1,4],process:[0,1,2,3,5,11,14],process_fil:1,produc:1,product:[1,2,3,5,7,9,11,14],profil:[1,5],pr
 ogram:[1,10],programmat:5,progress:[1,5,14],project:5,promin:7,prompt:0,propag:[1,2],propaget:1,proper:14,properli:[3,4],properti:[1,2,4,12],provid:[1,2,3,7,9,14,15],provide_context:[1,2],proxy_us:[1,12],proxyus:12,psycopg2:3,ptarget:1,publicli:7,publish:1,pull:[1,2],pull_funct:2,puppet:[3,10],pure:2,purpl:1,purpos:[7,14],push:[1,2,8],push_funct:2,pushing_task:2,put:[1,2,8],pwd:2,pydata:1,pyhiv:1,pypi:[6,13],python:[1,2,3,5,6,8,12,14],python_1:2,python_cal:[1,2],python_oper:1,pythonhost:8,pythonoper:[1,2],pythonpath:[3,12],qualif:12,qualifi:12,qualiti:[1,2],qubol:[1,6],qubole_conn_id:1,qubole_default:1,quboleoper:1,queri:[1,3,5],queu:[1,2,4],queue:[0,1],quick:[3,5],quickli:[9,13,14,15],quit:4,r39132:10,rabbitmq:[3,6],rais:[1,14],ran:0,randkei:12,random:1,randomli:2,rang:[0,1,11,14],rather:2,ratio:1,raw:1,reach:[1,2,3,4],reactiv:4,read:[1,3,4,12,14],readabl:[1,7,12],readi:[1,5,13],readm:3,readthedoc:14,ready_for_retri:1,real:3,realli:[1,12,14],realm:12,reason:[4,7,14],reboot:2,rebuil
 d:0,recap:5,receiv:[1,2,7,11],recent:[1,2],recip:10,recipi:7,recogn:1,recommend:[1,3,4,7],record:[0,1,2,14],recreat:1,recurs:1,red:1,redhat:3,redi:3,redirect:0,redistribut:7,redund:14,referenc:[2,14],reflect:[2,5,8,14],refrain:2,refresh:1,refresh_from_db:1,regard:[1,7],regardless:[0,1],regex:[0,1],regex_kei:1,regexp:0,regist:[8,9,12],regular:[1,11],reinit_frequ:12,rel:[1,11,14],relat:[1,2,3,4,6],relationship:2,relativedelta:1,relev:4,reli:[1,12],reliabl:1,remain:[7,12],rememb:[1,14],remot:[1,3,11],remote_base_log_fold:3,remote_full_path:1,remote_host:1,remote_log_conn_id:3,remote_port:1,remov:1,render:[0,1,2,8,14],render_templ:1,render_template_from_field:1,renew:[0,12],repeat:2,repl:12,replac:[1,7,12],report:[0,1],repositori:[3,14],repres:[1,2,4,7,11],represent:[1,15],reproduc:7,reproduct:7,request:[0,1,2],requir:[1,2,3,5,6,7,12],resembl:2,reset:0,resetdb:0,resid:2,resolv:1,resourc:[0,1,2,5],respect:[0,1,2,14],respons:[1,7],response_check:1,rest:1,restart:[2,3,4],restrict:12,restru
 cturedtext:2,result:[1,2,7,9,14],resum:0,retain:7,retri:[1,2,14],retriev:[1,2],retrieve_fil:1,retry_delai:[1,14],return_valu:1,reus:8,review:5,revis:7,riccomini:[5,10],rich:[0,2,5,15],rid:1,right:[1,2,7,9],rightmost:2,risk:7,roadmap:5,role:3,roll:5,room:1,room_id:1,root:[1,2],root_dag:1,round:[1,4],rout:12,routin:[1,2],row:[1,2],royalti:7,rule:1,run:[0,1,2,3,4,5],run_a:1,run_and_check:1,run_as_own:[1,12],run_cli:1,run_id:[0,1,11],runme_0:13,runnabl:[1,2],rwc:12,s3_conn_id:1,s3_default:1,s3_kei:1,s3connect:1,s3distcp:1,s3filetransferoper:2,s3hook:[1,3],s3keysensor:[1,6],s3prefixsensor:6,s3tohivetransf:1,safe_mod:1,sai:[1,2,4,11],said:2,samaccountnam:12,samba:[1,6],samba_conn_id:1,samba_default:1,same:[1,2,3,7,9,12],sampl:[1,3],sample_s:1,sandbox:3,save:[3,4,9,14],scala:1,scalabl:5,scan:[1,2],scene:[6,11],schedul:[0,1,2,3,4,5],schedule_interv:[1,2,4,11,14],scheduler_interv:4,scheduler_run:3,schema:[1,2,3],scratch:14,screen:14,screenshot:5,script:[1,2,3,4,5],script_begin_tag:1,script_l
 oc:1,search:[1,2],search_path:3,search_scop:12,second:[1,2,4,14],secret:[12,15],section:[1,2,3,4,7,12,13,14],secur:[1,2,5,7],see:[1,2,3,4,7,11,12,13,14,15],seen:[3,15],segment:1,select:[1,2,9,11,12],self:[1,8],sell:7,semicolon:1,send:[0,1,2,10],send_room_notif:1,sender:1,sensor:[1,2,4],sensor_queu:2,sent:[1,2,3,7],separ:[1,2,7,14],sequenti:[1,11,13],sequentialexecutor:[1,2,13],serial:[0,1],serv:[0,3,14],serve_log:[0,3],server:[0,1,3,8,12,13,14],servic:[0,1,2,3,6,7,11,12],session:[1,12],set:[0,1,2],set_depend:[1,14],set_downstream:[1,2,14],set_the_password:12,set_upstream:[1,2,14],setgid:3,setuid:3,setup:[3,12],sever:2,shall:7,shape:11,share:[1,2,3,7,8,9],shell:12,shellcmd:1,ship:[0,1],ship_dag:0,shop:7,shortcircuitoper:1,shortcut:[3,15],shortest:1,should:[1,2,3,4,7,8,11,13,14],shouldn:[1,2],show:[0,1,2,4,8,15],shown:1,sid:10,siddharth:10,side:[1,3],signal:1,similar:[2,3,5],simpl:[1,2,8,9,14],simplehttpoper:1,simpler:14,simplest:12,simpli:[1,2,8,11,12,14],simplifi:1,simul:14,simultan
 :2,sinc:[1,3,13,14],sing:1,singl:[0,1,2,14],site:12,size:[1,2],skew:8,skip:[0,1,2],sla:1,sla_miss_callback:1,slack:[1,6],slack_oper:1,slackapioper:1,slackapipostoper:[1,6],slackoper:2,slave:3,sleep:14,slightli:5,slot:[1,2],slowli:5,slug:12,small:[1,2],smaller:1,smallish:1,snap:5,sock:1,soft_fail:1,softwar:[5,7,10],sole:7,solut:5,some:[1,2,3,4,11,14,15],somehow:3,someth:[2,12,14],sometim:[1,2,14],somewhat:14,somewher:13,soon:[1,2,3,4,11],sort:2,sound:[2,14],sourc:[1,2,7,9,10,15],source_conn_id:1,source_project_dataset_t:1,space:5,span:15,spark:[1,2,5],sparkcmd:1,spawn:3,special:[1,2,4,7],specif:[0,1,2,3,4,7,11,14,15],specifi:[0,1,2,3,4,5,11,12,14],specific_valu:1,spent:15,split:1,split_column:1,sponsor:5,spring:10,sql:[1,2,3,6,9],sql_alchemy_conn:3,sql_alchemy_conn_cmd:3,sqlachemi:1,sqlalchemi:[1,2,3],sqlite:[1,13],sqlitehook:1,sqloper:2,sqlsensor:1,squeez:1,sscursor:1,ssh:[1,12],ssh_default:1,ssh_hook:1,sshexecuteoper:1,sshhook:1,sshpass:1,ssl:1,sslcert:1,sslmode:1,stabil:5,stabl:6,
 stack:[6,8],stage:[1,2],stai:11,stamp:11,stand:2,standalon:2,standard:[1,2],stanza:3,start:[0,1,2,3,4,5,6,10,11],start_dat:[0,1,2,4,11,14],stat:1,state:[0,1,2,4,7,11,14],state_d:1,statement:[1,2,7],static_babynam:1,static_babynames_partit:1,static_fold:8,static_path:1,static_url_path:8,statu:[0,1,2,3,4,5,7,11,13,14,15],stderr:0,stdout:[0,1,14],step:[1,3,14,15],steven:10,still:[1,4],sting:1,stop:[1,3],stoppag:7,storag:[1,3],store:[0,1,2,3,8,12,13],store_fil:1,storm:5,str:[1,8,11],straightforward:13,stream:[1,5],string:[0,1,2,3,14],string_data:1,structur:[1,5,14],style:1,sub:[0,1],sub_command:1,sub_commnad:1,sub_dag:[1,2],sub_part:1,subcommand:[1,3,11,14],subdag:[0,1],subdagoper:2,subdir:0,subdirectori:2,subject:[1,7],sublicens:7,submiss:[7,11],submit:[1,7],subpackag:[6,12],subpartit:1,subprocess:[1,11],subscrib:[2,10],subsect:[0,11],subsequ:[0,7,11],subset:1,subtl:2,subtract:1,subtre:12,succe:[1,2],succeed:[0,1,2,4,15],success:[0,1,2,4,5,11,14],successfulli:[1,2,14],suit:5,sum:2,summ
 it:10,sundai:11,super_us:12,supersed:7,superus:12,superuser_filt:12,suppli:[1,3,12],support:[0,1,3,6,7,12],sure:[1,2,3,4,12,14],surfac:14,surgeri:5,sync:[0,1,3,11],sync_to_db:1,synchron:[1,3],syntax:7,sysconfig:3,system:[1,2,3,7],syvineckruyk:10,tab:[1,12],tabl:[1,2,8],table_exist:1,table_nam:[1,2],tablenam:1,tag:[1,4],take:[1,2,3,10,12,13,14,15],talk:[1,10,14],tardif:10,target:[1,3,11],target_field:1,target_partition_s:1,target_tim:1,task:[0,1],task_id:[0,1,2,11,14],task_inst:[1,2],task_instance_key_str:1,task_or_task_list:1,task_param:0,task_regex:[0,1],task_stat:0,taskinst:[1,2],tbl:1,team:[1,12],tell:[0,1],templat:[0,1],template_fold:8,template_searchpath:[1,14],templated_command:14,templates_dict:1,templates_ext:1,temporari:[1,2],tenanc:5,tenant:12,term:7,termin:[2,7],test:[0,1,2,3,4,5,8,12],test_env:2,test_hql:1,test_mod:1,test_plugin:8,testabl:5,testview:8,text:[1,3,7,15],textfil:1,than:[1,2,5,14],thei:[0,1,2,3,5,6,8,9,11,14],them:[0,1,2,8,11,14],themselv:2,theoret:1,theori:[
 4,7],thereof:7,thi:[0,1,2,3,4,5,7,8,11,12,13,14,15],thin:1,thing:[1,2,4,14],think:[5,14],third:7,this_dag_will_be_discov:2,those:[1,2,3,7,12],though:[2,5],thousand:2,thread:1,three:[2,3],threshold:8,thrift:1,through:[1,2,6,7,11,13,14],throughout:1,ticket:[0,12],tighten:12,time:[1,2,3,4,8,11,12,14,15],timedelta:[1,2,4,11,14],timedeltasensor:[1,4],timefram:1,timeout:[0,1,2],timesensor:1,timestamp:[1,2,11],tip:[2,10],titl:[2,7],tls_ca_cert:1,tls_client_cert:1,tls_client_kei:1,tls_hostnam:1,tls_ssl_version:1,tmp:[1,2],tmp_dir:1,tmp_file:1,todai:1,togeth:[1,2],token:1,toler:[1,2],tomorrow:1,tomorrow_d:1,tomorrow_ds_nodash:1,too:[1,2,11],tool:[1,2,6,8],toolbox:8,top:[1,3,5],topic:3,tornado:0,tort:7,tour:[13,14],track:[2,7,14],tracker:1,trade:7,trademark:7,tradit:1,tradition:2,train:7,transact:1,transfer:[1,2,7],transform:7,translat:[1,7],transpar:15,travers:1,treat:[1,2],tree:[0,1,4,5,14],tree_view:1,tri:[1,12],trick:10,trigger:[0,1],trigger_dag:[0,11],trigger_dag_id:1,trigger_rul:[1,2],t
 riggerdagrunoper:[1,10],triggerrul:1,troubl:6,troubleshoot:[5,15],truncat:1,truth:1,ts_dim:1,ts_nodash:1,tsv:1,tty:1,tunnel:[1,12],tupl:1,turn:[2,12],tutori:[5,13],twice:1,two:[1,2,14],txt:1,type:[0,1,2,4,7,9,12],typic:[1,2],ubuntu:3,udf_config:1,uid:12,ultim:[1,14],unarchiv:1,unassign:2,unavail:3,unblock:11,undefin:12,under:[1,2,3,7,8,10],undergo:5,underli:8,underscor:3,understand:[1,4,8,14,15],undesir:11,unexpect:2,unexpectedli:2,union:7,uniqu:[1,14],unit:[3,5,12],unittest:1,unix:1,unless:[4,7],unpack:[1,2],unpaus:0,unset:1,unsound:[1,2],unsuccess:1,unsupport:1,until:[1,2,3,5],up_for_retri:1,updat:[1,2,4,12],updateonli:1,upgrad:0,upgradedb:0,upload:1,upon:[2,3,13],upstream:[0,1,2,4,11],upstream_fail:[1,2],upstream_list:1,upstream_task_id:1,uri:[1,2,3,12],url:[1,2,3,8,9,12],usag:[0,14],use_beelin:[1,12],user:[0,1,2,3,5,6,12,14],user_defined_macro:1,user_filt:12,user_name_attr:12,user_program_argu:1,usernam:[1,3,12],usr:3,usual:[2,3],utf8:1,utf:1,util:[1,5,13],uuid:1,val:[0,1],valid
 :[4,5,12],valu:[0,1,2,3,4,12,14,15],valuecheckoper:1,variabl:0,vast:1,verbal:7,verbos:[1,14],veri:[0,1,2,4,10,13,14],verifi:[2,4],version:[0,1,2,5,6,7],vertia:1,vertica:[1,6],vertica_conn_id:1,vertica_default:1,verticahook:1,verticaoper:1,verticaql:1,verticatohivetransf:1,via:[1,2,12],video:[1,10],view:[0,1,2,3,4,5,8],virtual:1,virtualenv:2,visibl:[1,2],visual:[5,8,9,14,15],vocabulari:2,volum:1,wai:[1,2,3,4,6,8,11,12,14,15],wait:[0,1,2],wait_for_downstream:[1,4],wait_for_empty_queu:2,walk:14,want:[1,2,3,4,8,11,12,14],warranti:7,wasn:4,wasnt:1,watch:[1,3,4],weapon:9,web:[0,1,2,3,5,8],webhdfs_conn_id:1,webhdfs_default:1,webhdfshook:1,webhdfssensor:1,webserv:[0,3,4,12,13,14],week:11,weekli:11,weight:1,well:[1,2,3,4,11],were:1,what:[1,2,4,5,6],whatev:[2,3,6],when:[1,2,3,4,5,8,9,11,12,14],whenev:1,where:[1,2,3,4,7,11,12,14,15],where_claus:1,wherev:[1,7],whether:[1,7,11],which:[0,1,2,3,4,7,11,12,13,14],whitelist:12,who:8,whole:[1,2,7],whom:7,whose:11,why:[4,5],wiedmer:10,wiki:10,wildcard:
 1,wildcard_kei:1,wildcard_match:1,win:1,window:11,winter:10,wire:2,within:[0,1,2,3,4,7],without:[0,1,2,7,11,12],won:[1,4,6],word:[2,3,11,15],work:[0,1,2,3,4,5,7,8,9,11,12,13],worker:[0,1,2,3,4,5,6,8,12,14],worker_timeout:0,workerclass:0,workflow:1,workload:8,worldwid:7,worri:14,worth:1,would:[1,2,3,4,5,11,12,14],wouldn:[1,4],wrap:14,wrapper:1,write:[3,5,7,8,9,14],write_disposit:1,write_empti:1,written:[1,7,14],wrong:14,www:[1,2,7],x01:1,xcom:1,xcom_al:1,xcom_pul:[1,2],xcom_push:[1,2],xcom_return_kei:1,xml:12,xst:12,yaml:2,year:[1,11],yearli:11,yellow:1,yesterdai:1,yesterday_d:1,yesterday_ds_nodash:1,yet:[1,2,5,12,15],you:[1,2,3,4,5,6,7,8,9,11,12,13,14,15],your:[1,2,3,4,5,6,7,8,9,11],yourself:1,youtub:1,yum:6,yvinec:10,yyyi:[0,1,2],yyyymmdd:1,zero:1,zip:[1,2],zip_dag:2,zip_dag_cont:2,zoom:2},titles:["Command Line Interface","API Reference","Concepts","Configuration","FAQ","Apache Airflow (incubating) Documentation","Installation","License","Plugins","Data Profiling","Project","Schedu
 ling & Triggers","Security","Quick Start","Tutorial","UI / Screenshots"],titleterms:{"default":[1,2,14],"function":2,"import":14,addit:2,adhoc:9,airflow:[5,6,8],apach:5,api:1,argument:[2,14],assign:2,authent:12,backend:3,backfil:14,baseoper:1,basesensoroper:1,beyond:5,bitshift:2,branch:2,build:8,celeri:3,chart:[9,15],cluster:2,code:15,command:[0,14],committ:10,commun:[1,3],composit:2,concept:2,configur:3,connect:[2,3],content:5,context:[2,15],contribut:[1,3],core:2,dag:[2,11,14,15],data:9,definit:14,depend:14,document:[2,5],durat:15,enabl:12,enterpris:12,exampl:[8,14],executor:1,extern:11,extra:6,faq:4,file:14,form:9,gantt:15,get:6,ghe:12,github:12,graph:15,histori:10,hook:[1,2],horizon:5,idea:2,incub:5,instal:6,instanc:[2,15],instanti:14,integr:3,interfac:[0,8],jinja:[2,14],keep:11,kerbero:12,ldap:12,licens:7,limit:12,line:[0,14],link:10,log:3,macro:1,manag:2,menu:15,meso:3,metadata:14,mind:11,model:1,modul:14,multi:12,next:[13,14],note:2,oper:[1,2],option:3,out:3,own:12,packag
 :[2,6],password:12,pipelin:14,plugin:8,polici:2,pool:2,principl:5,profil:9,project:10,queri:9,queue:2,quick:13,recap:14,refer:1,resourc:10,roadmap:10,roll:12,rule:2,run:[11,14],scale:3,schedul:11,scope:2,screenshot:[9,15],script:14,secur:12,set:[3,12,14],sla:2,start:13,subdag:2,systemd:3,task:[2,14,15],templat:[2,14],tenanc:12,test:14,top:8,tree:15,trigger:[2,11],tutori:14,undead:2,upstart:3,valid:14,variabl:[1,2,15],view:15,web:12,what:[8,13,14],why:8,workflow:2,xcom:2,your:12,zombi:2}})
\ No newline at end of file
+Search.setIndex({envversion:47,filenames:["cli","code","concepts","configuration","faq","index","installation","license","plugins","profiling","project","scheduler","security","start","tutorial","ui"],objects:{"airflow.contrib":{hooks:[1,1,0,"-"],operators:[1,1,0,"-"]},"airflow.contrib.executors.mesos_executor":{MesosExecutor:[1,0,1,""]},"airflow.contrib.hooks":{BigQueryHook:[1,0,1,""],CloudantHook:[1,0,1,""],FTPHook:[1,0,1,""],GoogleCloudStorageHook:[1,0,1,""],SSHHook:[1,0,1,""],VerticaHook:[1,0,1,""]},"airflow.contrib.hooks.BigQueryHook":{get_conn:[1,2,1,""],get_pandas_df:[1,2,1,""],get_service:[1,2,1,""],insert_rows:[1,2,1,""]},"airflow.contrib.hooks.CloudantHook":{db:[1,2,1,""]},"airflow.contrib.hooks.FTPHook":{close_conn:[1,2,1,""],create_directory:[1,2,1,""],delete_directory:[1,2,1,""],delete_file:[1,2,1,""],describe_directory:[1,2,1,""],get_conn:[1,2,1,""],list_directory:[1,2,1,""],retrieve_file:[1,2,1,""],store_file:[1,2,1,""]},"airflow.contrib.hooks.GoogleCloudStorageHook":
 {download:[1,2,1,""],get_conn:[1,2,1,""],upload:[1,2,1,""]},"airflow.contrib.hooks.SSHHook":{Popen:[1,2,1,""],check_output:[1,2,1,""],tunnel:[1,2,1,""]},"airflow.contrib.hooks.VerticaHook":{get_conn:[1,2,1,""]},"airflow.contrib.hooks.gcs_hook":{GoogleCloudStorageHook:[1,0,1,""]},"airflow.contrib.operators":{QuboleOperator:[1,0,1,""],SSHExecuteOperator:[1,0,1,""],VerticaOperator:[1,0,1,""],VerticaToHiveTransfer:[1,0,1,""]},"airflow.contrib.operators.bigquery_operator":{BigQueryOperator:[1,0,1,""]},"airflow.contrib.operators.bigquery_to_gcs":{BigQueryToCloudStorageOperator:[1,0,1,""]},"airflow.contrib.operators.gcs_download_operator":{GoogleCloudStorageDownloadOperator:[1,0,1,""]},"airflow.contrib.operators.hipchat_operator":{HipChatAPIOperator:[1,0,1,""],HipChatAPISendRoomNotificationOperator:[1,0,1,""]},"airflow.executors":{CeleryExecutor:[1,0,1,""],LocalExecutor:[1,0,1,""],SequentialExecutor:[1,0,1,""]},"airflow.hooks":{DbApiHook:[1,0,1,""],DruidHook:[1,0,1,""],HiveCliHook:[1,0,1,"
 "],HiveMetastoreHook:[1,0,1,""],HiveServer2Hook:[1,0,1,""],HttpHook:[1,0,1,""],MsSqlHook:[1,0,1,""],MySqlHook:[1,0,1,""],PostgresHook:[1,0,1,""],PrestoHook:[1,0,1,""],S3Hook:[1,0,1,""],SqliteHook:[1,0,1,""],WebHDFSHook:[1,0,1,""]},"airflow.hooks.DbApiHook":{bulk_dump:[1,2,1,""],bulk_load:[1,2,1,""],get_conn:[1,2,1,""],get_cursor:[1,2,1,""],get_first:[1,2,1,""],get_pandas_df:[1,2,1,""],get_records:[1,2,1,""],insert_rows:[1,2,1,""],run:[1,2,1,""]},"airflow.hooks.DruidHook":{construct_ingest_query:[1,2,1,""],get_conn:[1,2,1,""],load_from_hdfs:[1,2,1,""]},"airflow.hooks.HiveCliHook":{load_file:[1,2,1,""],run_cli:[1,2,1,""],test_hql:[1,2,1,""]},"airflow.hooks.HiveMetastoreHook":{check_for_named_partition:[1,2,1,""],check_for_partition:[1,2,1,""],get_databases:[1,2,1,""],get_metastore_client:[1,2,1,""],get_partitions:[1,2,1,""],get_table:[1,2,1,""],get_tables:[1,2,1,""],max_partition:[1,2,1,""],table_exists:[1,2,1,""]},"airflow.hooks.HiveServer2Hook":{get_pandas_df:[1,2,1,""],get_records:
 [1,2,1,""]},"airflow.hooks.HttpHook":{get_conn:[1,2,1,""],run:[1,2,1,""],run_and_check:[1,2,1,""]},"airflow.hooks.MsSqlHook":{get_conn:[1,2,1,""]},"airflow.hooks.MySqlHook":{bulk_load:[1,2,1,""],get_conn:[1,2,1,""]},"airflow.hooks.PrestoHook":{get_conn:[1,2,1,""],get_first:[1,2,1,""],get_pandas_df:[1,2,1,""],get_records:[1,2,1,""],run:[1,2,1,""]},"airflow.hooks.S3Hook":{check_for_bucket:[1,2,1,""],check_for_key:[1,2,1,""],check_for_prefix:[1,2,1,""],check_for_wildcard_key:[1,2,1,""],get_bucket:[1,2,1,""],get_conn:[1,2,1,""],get_key:[1,2,1,""],get_wildcard_key:[1,2,1,""],list_keys:[1,2,1,""],list_prefixes:[1,2,1,""],load_file:[1,2,1,""],load_string:[1,2,1,""]},"airflow.hooks.SqliteHook":{get_conn:[1,2,1,""]},"airflow.hooks.WebHDFSHook":{check_for_path:[1,2,1,""],get_conn:[1,2,1,""],load_file:[1,2,1,""]},"airflow.macros":{ds_add:[1,3,1,""],ds_format:[1,3,1,""],hive:[1,1,0,"-"],random:[1,3,1,""]},"airflow.macros.hive":{closest_ds_partition:[1,3,1,""],max_partition:[1,3,1,""]},"airflow.
 models":{BaseOperator:[1,0,1,""],Connection:[1,0,1,""],DAG:[1,0,1,""],DagBag:[1,0,1,""],TaskInstance:[1,0,1,""]},"airflow.models.BaseOperator":{clear:[1,2,1,""],dag:[1,4,1,""],detect_downstream_cycle:[1,2,1,""],downstream_list:[1,4,1,""],execute:[1,2,1,""],get_direct_relatives:[1,2,1,""],get_flat_relatives:[1,2,1,""],get_task_instances:[1,2,1,""],has_dag:[1,2,1,""],on_kill:[1,2,1,""],post_execute:[1,2,1,""],pre_execute:[1,2,1,""],prepare_template:[1,2,1,""],render_template:[1,2,1,""],render_template_from_field:[1,2,1,""],run:[1,2,1,""],schedule_interval:[1,4,1,""],set_downstream:[1,2,1,""],set_upstream:[1,2,1,""],upstream_list:[1,4,1,""],xcom_pull:[1,2,1,""],xcom_push:[1,2,1,""]},"airflow.models.Connection":{extra_dejson:[1,4,1,""]},"airflow.models.DAG":{add_task:[1,2,1,""],add_tasks:[1,2,1,""],cli:[1,2,1,""],concurrency_reached:[1,4,1,""],crawl_for_tasks:[1,2,1,""],create_dagrun:[1,2,1,""],filepath:[1,4,1,""],folder:[1,4,1,""],get_template_env:[1,2,1,""],is_paused:[1,4,1,""],latest
 _execution_date:[1,4,1,""],normalize_schedule:[1,2,1,""],run:[1,2,1,""],set_dependency:[1,2,1,""],sub_dag:[1,2,1,""],subdags:[1,4,1,""],tree_view:[1,2,1,""]},"airflow.models.DagBag":{bag_dag:[1,2,1,""],collect_dags:[1,2,1,""],dagbag_report:[1,2,1,""],get_dag:[1,2,1,""],kill_zombies:[1,2,1,""],process_file:[1,2,1,""],size:[1,2,1,""]},"airflow.models.TaskInstance":{are_dependencies_met:[1,2,1,""],are_dependents_done:[1,2,1,""],clear_xcom_data:[1,2,1,""],command:[1,2,1,""],current_state:[1,2,1,""],error:[1,2,1,""],evaluate_trigger_rule:[1,2,1,""],is_premature:[1,2,1,""],is_queueable:[1,2,1,""],is_runnable:[1,2,1,""],key:[1,4,1,""],pool_full:[1,2,1,""],ready_for_retry:[1,2,1,""],refresh_from_db:[1,2,1,""],run:[1,2,1,""],xcom_pull:[1,2,1,""],xcom_push:[1,2,1,""]},"airflow.operators":{BashOperator:[1,0,1,""],BranchPythonOperator:[1,0,1,""],DummyOperator:[1,0,1,""],EmailOperator:[1,0,1,""],ExternalTaskSensor:[1,0,1,""],GenericTransfer:[1,0,1,""],HdfsSensor:[1,0,1,""],Hive2SambaOperator:[1,
 0,1,""],HiveOperator:[1,0,1,""],HivePartitionSensor:[1,0,1,""],HiveToDruidTransfer:[1,0,1,""],HiveToMySqlTransfer:[1,0,1,""],HttpSensor:[1,0,1,""],MetastorePartitionSensor:[1,0,1,""],MsSqlOperator:[1,0,1,""],MsSqlToHiveTransfer:[1,0,1,""],MySqlOperator:[1,0,1,""],MySqlToHiveTransfer:[1,0,1,""],PostgresOperator:[1,0,1,""],PrestoCheckOperator:[1,0,1,""],PrestoIntervalCheckOperator:[1,0,1,""],PrestoValueCheckOperator:[1,0,1,""],PythonOperator:[1,0,1,""],S3KeySensor:[1,0,1,""],S3ToHiveTransfer:[1,0,1,""],ShortCircuitOperator:[1,0,1,""],SimpleHttpOperator:[1,0,1,""],SlackAPIOperator:[1,0,1,""],SlackAPIPostOperator:[1,0,1,""],SqlSensor:[1,0,1,""],TimeSensor:[1,0,1,""],TriggerDagRunOperator:[1,0,1,""],WebHdfsSensor:[1,0,1,""]},"airflow.operators.BashOperator":{execute:[1,2,1,""]},"airflow.operators.SlackAPIOperator":{construct_api_call_params:[1,2,1,""],execute:[1,2,1,""]},"airflow.operators.docker_operator":{DockerOperator:[1,0,1,""]},"airflow.operators.sensors":{BaseSensorOperator:[1,0,1
 ,""]},airflow:{executors:[1,1,0,"-"],hooks:[1,1,0,"-"],macros:[1,1,0,"-"],models:[1,1,0,"-"],operators:[1,1,0,"-"]}},objnames:{"0":["py","class","Python class"],"1":["py","module","Python module"],"2":["py","method","Python method"],"3":["py","function","Python function"],"4":["py","attribute","Python attribute"]},objtypes:{"0":"py:class","1":"py:module","2":"py:method","3":"py:function","4":"py:attribute"},terms:{"00am":1,"01t23":11,"10gb":1,"10pm":2,"128m":1,"2am":1,"5gb":1,"__class__":2,"__doc__":2,"__init__":[1,2],"__name__":[2,8],"_cmd":3,"_host":12,"abstract":[1,2,5,8],"boolean":[1,2],"byte":[1,4],"case":[1,2,4,11,12,14],"char":1,"class":[0,1,2,6,7,8,14],"default":[],"export":[1,13],"final":1,"float":1,"function":[],"import":[],"int":1,"long":1,"new":[1,2,4,8,11,12],"return":[0,1,2,8],"short":1,"static":[1,5,8],"super":[1,12],"switch":[3,12],"true":[1,2,4,12,14],"try":[1,2,3],"var":1,"while":[1,2,4,5,7,11,13,14,15],abbrevi:2,abil:2,abl:[1,3,4,12,13,14],about:[1,2,3,7,10,11,14]
 ,abov:[2,3,7,12,15],absenc:2,absolut:2,accept:[1,5,7],access:[0,1,2,3,8,12],access_logfil:0,access_token:15,accord:12,accordingli:2,account:[1,12],acquir:1,across:[3,15],act:[1,2,7,14],action:[1,15],activ:[1,2,4,8,11,12,14],actual:[1,2,4,14],acycl:[1,2,5],add:[1,2,3,7,12],add_task:1,addendum:7,addit:[],addition:12,addprinc:12,adhoc:[],adjust:[3,12],admin:[2,3,4,8,9,12,13],admin_view:8,advanc:[1,14],advantag:12,advis:[1,7],affect:2,after:[1,2,4,11],afterward:1,against:[1,2,4,7,12,14],agari:10,aggregate_db_message_job:2,aggregate_db_message_job_cmd:2,ago:4,agre:7,agreement:[2,7],airbnb:[1,2,10,14],airflow:[],airflow__:3,airflow__core__sql_alchemy_conn:3,airflow_config:3,airflow_conn_:[2,3],airflow_conn_postgres_mast:2,airflow_hom:[3,4,8,13],airflow_set:2,airflow_tmp_dir:1,airflowignor:1,airflowplugin:8,airflowtestplugin:8,aiyzngq:1,alchemi:3,alert:[1,2,8],align:4,all:[0,1,2,3,4,5,6,7,11,12,14],all_db:6,all_don:[1,2],all_fail:[1,2],all_success:[1,2],alleg:7,alloc:1,allow:[0,1,2,3,4,5,8
 ,9,11,12,13,14,15],allow_large_result:1,allowed_st:1,allowed_team:12,allowinsert:1,alon:7,along:[1,4,7,14],alongsid:[1,7,11],alreadi:1,alright:14,also:[1,2,3,4,6,7,8,11,12,14],alter:[1,2,3,4,12],altern:[1,11],altert:1,although:12,alwai:[1,2,15],amazon:3,amount:[1,2],analys:15,analyz:2,anand:10,anchor:1,ani:[1,2,3,4,7,14,15],annot:7,announc:10,anomali:8,anoth:[1,2,4,10],ansibl:3,anyon:12,anyth:[2,14],anytim:2,anywher:[1,2],aoen:10,apach:[],api:[],api_call_param:1,api_kei:15,api_param:1,api_vers:1,apikei:15,apiv2:1,app:1,app_id:1,appar:1,appear:[1,2,7],append:[1,3],appendix:7,appli:[1,2,6,7,11],applic:[1,7,8,9,12],appropri:[2,7,12],arbitrari:[2,5],architectur:5,archiv:[1,7],archive1:1,archive2:1,are_dependencies_met:1,are_dependents_don:1,aren:[1,2],arg:1,argument:[],aris:7,around:[1,5,14],arrai:[1,5,8],arthur:10,artwr:10,ascii:1,asf:5,assert:7,asset:1,assign:[],associ:[1,2,7,11],assum:[1,2,3,7,14],asterisk:12,async:6,atom:2,attach:[1,2,7],attach_to:1,attack:12,attempt:[0,1],attent:[1
 ,4],attr:1,attribut:[0,1,2,7],audit:8,auth:[1,12],auth_backend:12,auth_token:1,authen:12,authent:[],authmechan:1,author:[1,3,5,7,12,14,15],authorship:7,auto:4,autocommit:1,automat:[1,2,3],avail:[1,2,3,7,8,9],averag:1,avoid:2,awai:1,awar:2,azkaban:5,back:[1,4,8],backend:[],backfil:[],background:[1,14],backup:3,bag:1,bag_dag:1,balanc:2,bar:[1,2],bare:14,base64:4,base:[1,2,3,4,7,8,11],base_executor:[1,8],base_hook:[1,8],base_url:1,basedn:12,baseexecutor:[1,8],basehook:[1,2,8],baseview:8,bash:[1,2,14],bash_1:2,bash_command:[1,2,14],bash_command_to_run:3,bash_oper:[1,14],bash_queu:14,bashoper:[1,2,14],basi:7,basic:[2,6,8,9,14],bcrypt:12,beauchemin:10,becaus:[1,2],becom:[1,2,5,8,14],beelin:1,been:[1,2,3,7,11,12],befor:[0,1,2,4,12],begin:[2,11],behalf:7,behav:1,behavior:[1,2],behind:[1,3,6,11],belong:[1,12],below:[1,7,8,13,14],benefici:7,best:[1,10],better:14,between:[1,2,14],beyond:[],bigqueri:1,bigquery_conn_id:1,bigquery_default:1,bigquery_oper:1,bigquery_to_gc:1,bigqueryhook:1,bigquery
 oper:1,bigquerytocloudstorageoper:1,bin:2,bind:7,bind_password:12,bind_us:12,bit:3,blob:[1,14],block:[1,2,14,15],blog:[7,10],blueprint:8,bodi:1,boilerpl:7,bolk:10,bolkedebruin:10,bonu:1,bool:1,boot:3,both:[1,4],bother:14,boto:1,bottleneck:15,boundary_queri:1,box:[1,3,13],bql:1,bracket:[7,14],branch:[],branch_fals:2,branchpythonoper:[1,2],broker:[0,3],broker_api:0,broker_url:3,brought:10,brows:[2,4],bruin:10,bucket:[1,3],bucket_kei:1,bucket_nam:1,buffer:1,bug:3,build:[],built:[1,2,3,5,8,9,14],bulk:[4,15],bulk_dump:1,bulk_load:1,bump:2,bundl:1,burn:0,but_this_dag_will_not:2,c12318391:1,cacert:12,cach:12,call:[1,2,3,14],callabl:1,callback:12,can:[1,2,3,4,5,6,8,9,11,12,13,14,15],cannot:[1,2,7,14],capabl:8,capac:2,card:1,care:[0,3],carri:[2,7],cast:1,cat:1,categori:8,caus:[4,7],celeri:[],celery_result_backend:3,celeryexecutor:[1,2,3,4,6,11],cell:1,central:2,cert:1,certain:[1,2],certif:1,cfg:[1,2,3,4,11,12,13,14],chain:2,chang:[3,5,7,11,12,13,14],channel:[1,10],charact:7,character:2,charg
 :7,charset:1,chart:[],chat:10,check:[0,1,2,4],check_for_bucket:1,check_for_kei:1,check_for_named_partit:1,check_for_partit:1,check_for_path:1,check_for_prefix:1,check_for_wildcard_kei:1,check_head:1,check_oper:1,check_output:1,checkoper:1,chef:[3,10],child:[1,2],child_dag:2,child_dag_nam:2,chmod:12,choic:[0,1,14],choos:[1,2,7],chri:10,circuit:1,claim:7,clariti:5,claus:1,clean:1,cleanup:1,clear:[0,1,2,4,11,15],clear_xcom_data:1,cli:[1,2,3,4,11],click:[4,12,14,15],client:[1,6,12],client_id:12,client_secret:12,close:[1,4,12],close_conn:1,closest:1,closest_ds_partit:1,cloud:[1,3,6],cloudant:[1,6],cloudant_conn_id:1,cloudant_default:1,cloudanthook:1,cluster:[],cluster_label:1,cmd:1,cmdline:1,code:[],col:1,collabor:5,collect:[1,2,8],collect_dag:1,color:1,column:1,com:[1,2,12,14],combin:[2,7],come:[1,11],comma:[0,1,2],command_typ:1,comment:7,commerci:7,commit:[1,10,12],commit_everi:1,committ:[],common:[2,3,4,7,10,14],commonli:1,compani:8,compar:5,compil:[4,7],complet:[1,2,3,5],complex:[1,2
 ,5,8],compli:7,complianc:7,complic:14,compon:8,compos:[2,12,14],comprehens:15,compress:1,comput:[1,7],conceptu:2,concern:2,concur:4,concurr:[0,1,4],concurrency_reach:1,condit:[1,2,6,7],conf:[0,1,3],config:[8,12],configur:[],confirm:[0,4,11],confus:[1,2,4],conjunct:[0,2,13],conn_id:[1,2,3],conn_typ:1,connect:[],connect_timeout:1,connecton:9,consequenti:7,consid:[1,2,3],consider:1,consist:[1,2,5,7],conspicu:7,constant:[1,2],constitu:2,constitut:7,constraint:1,constru:7,construct:1,construct_api_call_param:1,construct_ingest_queri:1,constructor:[1,14],contact:[3,7],contain:[1,2,3,7,11,12,14,15],content:[],context:[],continu:[1,5],contract:7,contrib:[1,2,12],contributor:[7,10],contributori:7,control:[1,2,7,15],conveni:[1,2],convent:2,convers:7,cool:1,coordin:1,copi:[1,3,7,12],copyright:[7,12],core:[],correct:2,correpsond:2,correspond:[1,2,3,4],could:[1,2,14],count:1,counterclaim:7,cours:12,cover:11,cpu:[1,8],craft:1,crawl:[1,11],crawl_for_task:1,creat:[1,2,3,4,8,9,11,12,13,14,15],create
 _dagrun:1,create_directori:1,creation:1,credenti:12,credit:12,criccomini:10,criteria:[1,2],criterion:1,critic:[1,2],cron:[1,4,11],cross:[2,7,14],crt:12,crypto:[3,4,6],csv:1,curli:14,current:[1,15],current_st:1,cursor:1,custom:8,customari:7,cycl:[1,2,14],daemon:[0,1,3],dag:[],dag_1:2,dag_2:2,dag_fold:[1,2],dag_id:[0,1,2,14],dag_md:2,dag_run:1,dag_run_obj:1,dag_stat:0,dagbag:[0,1],dagbag_report:1,dagfil:2,dagrun:[0,1,4],dagrun_timeout:1,dags_fold:3,dai:[1,11,13,14],daili:[1,4,11],damag:7,dan:[1,10],data:[],data_profiler_filt:12,databas:[0,1,2,3,5,6,8,9,13,14],datafram:1,dataset:1,datasourc:1,date:[0,1,2,7,11,14],date_filter_column:1,datestamp:1,datetim:[1,2,4,11,14],dateutil:1,davydov:10,days_back:1,db_tabl:1,db_tap_id:1,db_update_kei:1,db_update_mod:1,dbapi_hook:1,dbapihook:1,dbexportcmd:1,dbimportcmd:1,dbtap_id:1,dbtapquerycmd:1,deal:4,debug:[0,1,14],decid:[1,3],decis:5,declar:1,deep:1,def:[1,2,8],default_arg:[1,2,4,14],default_login:12,default_queu:2,defend:7,defer:2,deferred_op:2,
 defin:[1,2,3,4,5,7,8,11,12,14],definit:[],del:1,delai:[1,4],delegate_to:1,delet:[1,2,11,15],delete_directori:1,delete_fil:1,deliber:[1,7],delimit:[0,1,2],demonstr:2,dep:1,depend:[],depends_on_past:[0,1,2,4,14],deploi:8,deploy:8,depth:14,deriv:[1,3,7,8],descend:2,describ:[1,2,7],describe_directori:1,descript:[1,2,7],deseri:[0,1,2],deserialize_json:2,design:[2,7,11],desir:[1,14],destin:1,destination_cloud_storage_uri:1,destination_conn_id:1,destination_dataset_t:1,destination_filepath:1,destination_t:1,detail:[1,2,4,12,15],detect:[1,8],detect_downstream_cycl:1,determin:[1,2,7],dev:[3,6,10],devel:6,devel_hadoop:6,develop:[0,1,12,14],deviat:1,dialog:11,dict:[0,1,2],dictionari:[1,2,14],did:14,die:2,differ:[1,2,7,8,14,15],dimens:1,direct:[1,2,3,5,7],directli:[1,2,4,9],directori:[0,1,2,3,12,14],dirti:13,disabl:1,disclaim:[],discuss:7,disk:1,displai:[3,7,11,14],disregard:[1,14],distinct:1,distribut:[1,2,6,7],do_pickl:0,doc:[1,2,4,14],doc_json:2,doc_md:2,doc_rst:2,doc_yaml:2,docker:[1,10],do
 cker_oper:1,docker_url:1,dockeroper:[1,2],document:[],doe:[0,1,2,3,4,5,6,7,12,14],does_not_exist:1,doesn:[1,2,14],domain:12,don:[1,2,3,6,7,11],done:[1,2,11,12,14],donot_pickl:[0,1],door:2,dot:[1,2],doubl:[1,3,14],down:[0,2],download:[1,3],downstream:[0,1,2,11],downstream_list:1,downstream_task_id:1,drive:3,driven:[1,8],driver:3,drop:[1,8],druid:[1,6],druid_datasourc:1,druid_ingest_conn_id:1,druid_ingest_default:1,druid_query_conn_id:1,druid_query_default:1,druidhook:1,dry:0,dry_run:0,ds_add:[1,14],ds_format:1,ds_nodash:1,dttm:1,dubiou:1,dummi:[1,2,8],dummy_1:2,dummy_oper:2,dummy_task:2,dummyoper:[1,2],dump:1,durat:[],dure:[3,15],dynam:[1,2,4,5],each:[1,2,4,7,10,11,14],earlier:14,easi:[2,3,5,8,9,12,15],easier:[1,7],easiest:6,easili:[1,2,5,9,14],echo:[2,14],ecosystem:8,edit:[3,15],editori:7,effect:[1,2],effici:1,effort:5,egg:3,either:[1,2,3,7,12,13],elabor:7,elaps:1,elast:1,electron:7,eleg:5,element:3,els:13,elsewher:2,email:[1,2,10,12,14],email_on_failur:[1,14],email_on_retri:[1,14],
 emailoper:[1,2],emit:[2,14],empti:[1,2],enabl:[],enclos:7,encod:[1,4],encrypt:[1,3,4,6,12],encrypt_s3_log:3,end:[1,7,11],end_dat:[0,1,14],endors:5,endpoint:1,enforc:[2,4],engin:[1,4,5],enterpris:[],entir:[0,4],entiti:7,entri:[1,11],env:[1,2],environ:[1,2,3,5,6,9,11,14,15],ep_data_pipeline_db_msg_agg:2,equival:[2,6,14],era:4,error:[0,1],error_logfil:0,especi:[2,4,14],essenti:1,etc:[1,2,3,12],eval:1,evalu:[1,2,4,11,14],evaluate_trigger_rul:1,even:[1,2,7,9],event:[2,7,14],eventlet:0,ever:1,everi:[1,2,11,12,14],everyon:14,everyth:[14,15],exactli:[1,15],exampl:[],example1:13,example_bash_oper:13,example_dag:[2,14],example_team_1:12,example_team_2:12,exce:1,except:[1,2,4,7,14],exchang:[2,5],exclud:[0,7],exclude_subdag:0,exclus:[7,11],execut:[0,1,2,3,4,5,7,11,14],execution_d:[0,1,2,14],execution_date_fn:1,execution_delta:1,execution_timeout:[1,2],exercis:7,exhaust:3,exist:[0,1,2,3,4,14],exit:[0,12],expect:[1,2,4,5],expir:1,explain:1,explan:14,explicit:5,explicit_op:2,explicitli:[1,2,7,12,1
 4],export_dir:1,export_format:1,expos:[1,2,3,8,12],express:[1,4,7,11],ext:1,extend:5,extens:[1,5],extern:[],external_dag_id:1,external_task_id:1,external_trigg:1,externaltasksensor:1,extra:[],extra_dejson:1,extra_opt:1,extract:1,extract_queri:1,fact:2,factori:2,fail:[0,1,2,3,4,11,14,15],failur:[1,3,4,7],fairli:13,fall:10,fals:[0,1,3,11,14],familiar:[2,14],faq:[],fast:13,fault:[1,2],featur:[1,2,3,6,8,14,15],feb:12,fee:7,fernet:4,fernet_kei:4,fetch:[1,3],few:[1,3,9,13,14],field:[1,7],field_delimit:1,field_dict:1,fields_terminated_bi:1,fifti:7,figur:[1,2,11],file1:1,file2:1,file:[],filenam:[1,2],filepath:1,filestatu:1,filesystem:3,fill:[1,2,9,12],filter:[0,1,2,12],filter_by_own:12,find:[1,3,4,14,15],fine:14,finish:1,fire:[1,2,3],first:[0,1,2,3,4,10,11,13,14],fit:[1,5,7],fix:[1,11],flag:[1,11],flag_upstream_fail:1,flask:[0,8,9],flask_admin:8,flask_blueprint:8,flask_login:12,flat:1,flavor:8,flexibl:[1,14],flower:[0,3],folder:[1,3,8,11,13,14],follow:[1,2,3,5,7,12,14],foo:[1,2],foo_dag_id:
 4,foobar:3,forc:[0,1],force_pul:1,foreground:0,foreign:1,form:[],format:[1,2,3,7],forward:[1,4],found:[1,3,12],foundat:[5,7,10,13],four:2,framework:[3,8],free:[2,7],frm:1,from:[0,1,2,3,4,5,7,8,9,10,11,12,13,14,15],ftp:1,ftp_conn_id:1,ftp_default:1,ftphook:1,full:[1,10],full_filepath:1,fulli:[4,5,12],functiontyp:1,fundament:14,further:[5,13,14],futur:[1,11],galaxi:8,gantt:[],garag:2,gate:12,gcp_api:6,gcp_api_base_hook:1,gcs_download_oper:1,gcs_hook:1,gender:1,gener:[0,1,2,5,7,8,12,14,15],generictransf:1,get:[],get_bucket:1,get_conn:1,get_connect:2,get_cursor:1,get_dag:1,get_databas:1,get_direct_rel:1,get_first:1,get_flat_rel:1,get_kei:1,get_metastore_cli:1,get_pandas_df:1,get_partit:1,get_partitions_by_filt:1,get_record:1,get_servic:1,get_tabl:1,get_task_inst:1,get_template_context:1,get_template_env:1,get_wildcard_kei:1,gevent:0,ghe:[],ghe_oauth:12,ghost:1,git:3,github:[],github_enterpris:12,github_enterprise_auth:12,githubusercont:1,gitignor:1,gitter:10,give:[2,7],given:[1,2],glanc
 :15,global:[1,2,4],goal:14,goe:[1,4,14],good:2,goodby:2,goodwil:7,googl:[1,3,6],google_cloud_storage_conn_id:1,google_cloud_storage_default:1,googlecloudbasehook:1,googlecloudstoragedownloadoper:1,googlecloudstoragehook:1,govern:7,grab:1,grade:3,grai:1,grant:7,graph:[],great:[2,3],greater:1,green:1,grossli:7,group:[1,3,12],gunicorn:[0,6],hack:1,had:[1,4],hadoop:[6,10,12],hadoop_dependency_coordin:1,hadoopcmd:1,hand:[12,13],handi:10,handl:[1,2,3],happen:[2,11,14],hard:[1,2],harmless:7,has_dag:1,hash:[1,12],hasn:[1,2],have:[1,2,3,4,5,6,7,8,9,11,12,14],haven:[1,2,14],hdf:[1,2,6],hdfs_conn_id:1,hdfs_default:1,hdfs_path:1,hdfscli:1,hdfssensor:1,head:[13,14],header:1,heartbeat:[1,2],heavier:1,hello:[2,8],help:[0,7,8,12],here:[1,2,3,4,6,8,10,11,13,14,15],herebi:7,herein:7,hex:1,hidden:15,hierarchi:[1,14],high:1,highchart:9,higher:1,highli:3,highlight:14,hipchat:1,hipchat_oper:1,hipchatapioper:1,hipchatapisendroomnotificationoper:1,histori:[],hit:[1,2],hive2mysqloper:1,hive2sambaoper:[1,6],
 hive:[1,2,3,6,8,12],hive_cli_conn_id:1,hive_cli_default:1,hive_cli_param:1,hive_conf:1,hive_conn_id:1,hive_t:1,hiveclihook:1,hivecmd:1,hiveconf:1,hiveconf_jinja_transl:1,hivemetastorehook:1,hiveoper:[1,2,3,12],hivepartitionsensor:[1,2],hivesensoroper:1,hiveserver2:1,hiveserver2_conn_id:1,hiveserver2_default:1,hiveserver2hook:1,hivetodruidtransf:1,hivetomysqltransf:1,hold:7,home:13,homogen:3,hope:11,horribl:14,host:[1,3,12],hostnam:[0,1,2,3],hour:[1,2,4,11],hourli:[1,4,11],hous:2,how:[1,2,3,4,7,8,11,14,15],howev:[2,7,12],hql:1,html:[1,8,14],html_content:1,http:[1,2,7,8,12,14],http_conn_id:1,http_default:1,httphook:1,httpoper:2,httpsensor:1,human:1,icon:1,icon_url:1,idea:[],idempot:1,identif:7,identifi:[0,1,7,14,15],ignor:[0,1],ignore_depend:[0,1],ignore_depends_on_past:[0,1],ignore_first_depends_on_past:[0,1],iii:7,imag:1,immedi:[1,2],imper:3,imperson:12,implement:[1,2],impli:[1,7],improv:7,impyla:1,inabl:7,inact:4,inc:7,incident:7,includ:[0,1,2,3,7,11,14],include_adhoc:[0,1],include
 _downstream:1,include_exampl:1,include_prior_d:1,include_queu:1,include_upstream:1,inclus:7,incorpor:7,incur:7,indemn:7,indemnifi:7,independ:[1,2],index:1,indic:[1,2,5,7],indirect:7,individu:[1,7,11,14],ineffici:1,infer:[1,2],inferred_op:2,infil:1,infin:5,info:1,inform:[1,2,3,7,11,12,14],infrastructur:[1,5],infring:7,ingest:1,inherit:[1,14],init:3,initctl:3,initdb:[0,3,13],initi:[0,3,12,13],initialis:1,inject:8,inlin:1,input:1,input_format:1,insecur:12,insecurecli:1,insert:[1,2],insert_row:1,insid:[1,2],inspect:[11,13],instal:[],instanc:[],instanti:[],instati:14,instead:[0,1,4],insteadnhttp:1,institut:7,instruct:2,insur:1,integr:[],integrate_plugin:[],intention:7,inter:2,interact:[1,2,3,8,9],interest:[1,14],intergr:8,intern:0,interpret:[1,2],interv:[1,4],intervalcheckoper:1,intoth:1,introduc:4,intuit:14,invari:[1,2],invok:1,ipython:2,irrevoc:7,is_paus:1,is_prematur:1,is_queueabl:1,is_runn:1,ish:1,isn:[1,2,4],isoformat:1,issu:[1,2,5,7,10],item:2,iter:1,itself:[1,2,12,14],januari:[7,1
 1],jar:1,java:1,jdbc:[1,6],jeremiah:10,jinja2:1,jinja:[],jinja_env:1,jlowin:10,job:[0,1,2,3,4,11,13,15],job_id:1,jobtrack:1,join:[2,10],json:[0,1,2],jupyt:2,just:[0,1,2,4,8,14],kadmin:12,kapil:3,keep:[],kei:[0,1,2,3,4,12,15],kerber:[6,12],kerbero:[],kernel:1,key1:1,key_fil:1,keytab:[0,12],keyword:1,kick:[3,11],kill:[1,2],kill_zombi:1,kind:7,know:[3,14],knowledg:8,known:6,known_host:1,kruyk:10,kwarg:1,kwd:1,l447:1,lab:1,label:[1,9],lai:13,lambda:[1,2],land:[1,8],languag:[1,7,14],larg:1,larger:1,last:1,late:15,later:9,latest:[0,1,6,14],latest_d:1,latest_execution_d:1,latin_stat:1,launch:12,law:7,lawsuit:7,layout:14,ldap3:12,ldap:[],ldap_auth:12,ldap_ca:12,lead:[1,2],lean:5,least:[1,2,7],leav:1,led:2,left:[2,12],legaci:4,legal:7,len:1,leo:10,less:1,lesson:10,let:[0,1,2,3,9,14,15],level:[1,2,5,11,12],leverag:[1,2,14],liabil:7,liabl:7,lib:[1,2,3],libffi:3,librari:[1,2,3,5,8,14],libz:2,licens:[],licensor:7,life:2,light:[1,2],lighter:1,lightweight:2,like:[1,2,3,6,8,12,14],limit:[],link:[],
 list:[0,1,2,3,4,6,7,8,10,12,14,15],list_dag:[0,4,14],list_directori:1,list_kei:1,list_prefix:1,list_task:[0,4,14],listen:2,lit:1,liter:1,litig:7,live:1,load:[0,1,2],load_fil:1,load_from_hdf:1,load_str:1,local:[0,1,2,3,12,14],local_executor:1,local_full_path_or_buff:1,local_infil:1,local_port:1,localexecutor:[0,1,2,3,11],localhost:2,locat:[0,1,2,3,12,14],lock:1,lock_for_upd:1,log:[],log_fil:0,logfil:0,loggin:1,loggingmixin:1,logic:[1,2,14],login:[1,2,3,12],logist:8,longer:4,look:[0,1,2,4,5,8,10,11,14],loss:7,lot:11,lowin:10,machin:[1,2,3,12],made:[1,2,7],mai:[1,2,3,4,7,11,14],mail:[7,10],main:[1,2,4,8],main_dag:2,maintain:[1,5,10],make:[1,2,3,4,5,7,12,14,15],malfunct:7,malik:3,man:6,manag:[],mani:[0,1,2,4,8,14,15],manner:5,manual:[1,2,3,11],map:1,mapr:1,mark:[0,1,2,7,11],mark_success:[0,1,11],markdown:2,markup:1,master:[1,2,14],match:[0,1,2],matter:1,max:[1,10],max_active_run:[1,4],max_partit:1,maxim:10,maxime_beauchemin:0,maximum:1,mayb:2,mean:[1,2,3,4,7,12,14],meaning:1,meant:[1,4]
 ,mechan:[1,7,12],media:7,medium:7,meet:[1,7],mem_limit:1,member:12,memberof:12,memori:1,menu:[],menu_link:8,menulink:8,merchant:7,mere:7,merg:14,meso:[],mesos_executor:1,mesosexecutor:[1,3,11],messag:[1,2,5],message_format:1,met:[1,3,4,11],metadata:[],metastor:1,metastore_conn_id:1,metastore_default:1,metastore_mysql:1,metastorepartitionsensor:1,method:[1,2],metric:[1,8],metric_spec:1,metrics_threshold:1,microsoft:[1,6],midnight:[4,11],might:[1,2,4,11,14],migrat:12,mime:1,mime_typ:1,min:4,mind:[],minimum:[1,6,11],minut:[2,4,11,14],mismatch:2,miss:[1,2],mistercrunch:10,mix:14,mkdir:2,mlsd:1,mode:[0,1,14],modif:7,modifi:[1,2,3,7,9],modul:[],modular:5,moment:4,monitor:[2,3,5,8,11,15],monospac:2,month:[4,11],monthli:[4,11],more:[1,2,3,5,7,8,11,12,14,15],morn:11,most:[1,2,12,14,15],mostli:[1,5,11],mount:[1,3],move:[1,2,4,5],msg:1,mssql:[1,6],mssql_conn_id:1,mssql_default:1,mssqlhook:1,mssqloper:1,mssqltohivetransf:1,much:[1,2,11],multi:[],multipart:1,multipart_byt:1,multipl:[1,2],multipl
 i:1,multiprocess:1,must:[1,2,3,4,7,12,14],mutat:2,my_conn_str:3,my_dag1:2,my_dag2:2,my_dag:2,my_databas:1,my_dict_var:1,my_funct:2,my_param:14,my_tabl:1,my_useful_packag:2,my_var:1,myarg:1,mypackag:12,mys3conn:3,mysql:[1,2,3,6],mysql_conn_id:1,mysql_default:1,mysql_postoper:1,mysql_preoper:1,mysql_tabl:1,mysqldb:1,mysqlhook:1,mysqloper:[1,3],mysqltohivetransf:1,name:[1,2,3,7,8,12],namedhivepartitionsensor:1,namespac:[1,8],nav:12,navig:12,ndfor:14,necessarili:[5,7],need:[1,2,3,4,5,6,8,11,12,13,14],neg:[1,11],neglig:7,neighbour:1,nest:[1,14],network:[1,12],network_mod:1,never:[0,1,4],new_user_email:12,new_user_nam:12,newli:5,newobject:1,next:[],nhere:1,nice:2,night:2,nlst:1,no_confirm:0,no_host_key_check:1,node:[0,1,2],non:[1,3,7],none:[1,2,8,11],noram:12,norandkei:12,normal:[1,2,7],normalize_schedul:1,notat:1,note:[],notebook:2,noth:[1,7],notic:[2,7,14],notif:1,notifi:1,notwithstand:7,nov:10,now:[1,2,3,4,12,14],nuanc:2,num:1,num_run:0,num_shard:1,number:[0,1,2,3,5],oauth2:12,oauth_ca
 llback_rout:12,oauth_key_from_github_enterpris:12,oauth_secret_from_github_enterpris:12,obj:1,object:[0,1,2,3,4,7,8,11,14],objectclass:12,oblig:7,observ:1,obtain:7,occur:[1,2],occurr:1,octet:1,off:[1,3,11],offer:[7,8],offici:[1,10],offset:1,often:[1,2],on_failure_callback:1,on_kil:1,on_retry_callback:1,on_success_callback:1,onboard:4,onc:[1,2,3,4,11,14],one_fail:[1,2],one_success:[1,2],onli:[0,1,2,3,4,6,7,11,12,13],only_fail:0,only_if_upd:1,only_run:0,oozi:5,op1:2,op2:2,op3:2,op4:2,op_arg:1,op_kwarg:1,open:[1,2,4,10,12],oper:[],oppos:[1,2],optim:1,option:[],orchestr:[1,5],order:[1,2,3,11,12],org:[1,7,8,10,14],organ:[2,8],organiz:12,origin:7,orm:1,other:[1,2,3,5,7,9,11,14],otherwis:[1,7,14],our:14,out:[],outag:8,outgrow:13,outlier:15,output:[1,14],output_encod:1,output_format:1,outsid:11,outstand:7,over:[0,1,2,3,13,15],overlap:15,overrid:[0,1,14],overridden:[1,4],overview:15,overwhelm:2,overwrit:1,own:[],owner:[1,2,7,12,14],ownership:7,packag:[],package1:2,page:[2,4,7,10,15],pai:[1,4
 ],pair:[1,15],panda:1,parallel:[1,2,3,13],param:[0,1,14],paramet:[0,1,2,3,4,9,14],parameter:[2,5,9],parent:[1,2],parent_dag:[1,2],parent_dag_nam:2,parent_modul:1,pars:[1,4,8,14],part:[1,2,7,9,12],parti:7,particular:7,partit:[1,2],partition_nam:1,partition_spec:1,partli:4,pass:[1,2,4,8,14],pass_valu:1,passwd:15,password:[],password_auth:12,passwordus:12,past:[1,2,4,11,15],patent:7,path2:1,path:[1,2,3],patrick:10,patrickleotardif:10,pattern:[1,2],paus:[0,1],payload:1,pem:1,peopl:[1,8,14],pep:1,percent:7,perfect:2,perform:[0,1,2,3,5,7,15],perhap:[2,15],period:[1,2,4,11,14],permiss:[7,12],perpetu:7,persist:[1,11],perspect:2,pertain:7,phase:8,physic:1,pick:[2,3,9],pickl:[0,1,2,3],picklabl:1,pickle_id:1,pickleabl:1,pid:[0,13],pig:[1,2],pigcmd:1,piggi:8,pigoper:1,pin_100:1,pip:[2,6,13],pipelin:[],pitfal:10,place:[1,2,7,14],placehold:1,plain:[1,3],plan:4,platform:[1,2,5,6],pleas:[2,3,10,12,14],plug:8,plugin:[],pluginexecutor:8,pluginhook:8,pluginoper:8,plugins_manag:8,png:1,point:[1,2,3,4,1
 3,14],poke:1,poke_interv:1,polymorph:1,pool:[],pool_ful:1,popen:1,popul:14,port:[0,1,3,12,13],posit:[0,1],possibl:[0,1,2,3,7,12,13,14,15],post:[1,10],post_execut:1,postgr:[1,2,3,6],postgres_conn_id:1,postgres_default:1,postgres_mast:2,postgreshook:1,postgresoper:1,potenti:2,power:[2,5,7,14],practic:[1,10],pre:4,pre_execut:1,preced:[1,3,14],prefer:[7,11,13],prefix:[1,2,3],preoper:1,prepar:[2,7],prepare_templ:1,prepped_request:1,present:1,preset:11,presto:1,presto_conn_id:1,presto_default:1,prestocheckoper:1,prestocmd:1,prestohook:1,prestointervalcheckoper:1,prestotomysqloper:2,prestovaluecheckoper:1,pretti:14,prevent:[1,2,12],previou:[0,1,2,4,14],previous:[2,4],primari:1,primit:1,princip:[0,2,12],print:[0,1,2,7,14],print_dat:14,print_head:1,prior:1,priorit:2,prioriti:[1,2],priority_weight:[1,2,14],privaci:7,probabl:[3,4,12],problem:2,problemat:2,proce:[1,4],process:[0,1,2,3,5,11,14],process_fil:1,produc:1,product:[1,2,3,5,7,9,11,14],profil:[],program:[1,10],programmat:5,progress:[1,5
 ,14],project:[],promin:7,prompt:0,propag:[1,2],propaget:1,proper:14,properli:[3,4],properti:[1,2,4,12],provid:[1,2,3,7,9,14,15],provide_context:[1,2],proxy_us:[1,12],proxyus:12,psycopg2:3,ptarget:1,publicli:7,publish:1,pull:[1,2],pull_funct:2,puppet:[3,10],pure:2,purpl:1,purpos:[7,14],push:[1,2,8],push_funct:2,pushing_task:2,put:[1,2,8],pwd:2,pydata:1,pyhiv:1,pypi:[6,13],python:[1,2,3,5,6,8,12,14],python_1:2,python_cal:[1,2],python_oper:1,pythonhost:8,pythonoper:[1,2],pythonpath:[3,12],qualif:12,qualifi:12,qualiti:[1,2],qubol:[1,6],qubole_conn_id:1,qubole_default:1,quboleoper:1,queri:[],queu:[1,2,4],queue:[],quick:[],quickli:[9,13,14,15],quit:4,r39132:10,rabbitmq:[3,6],rais:[1,14],ran:0,randkei:12,random:1,randomli:2,rang:[0,1,11,14],rather:2,ratio:1,raw:1,reach:[1,2,3,4],reactiv:4,read:[1,3,4,12,14],readabl:[1,7,12],readi:[1,5,13],readm:3,readthedoc:14,ready_for_retri:1,real:3,realli:[1,12,14],realm:12,reason:[4,7,14],reboot:2,rebuild:0,recap:[],receiv:[1,2,7,11],recent:[1,2],recip
 :10,recipi:7,recogn:1,recommend:[1,3,4,7],record:[0,1,2,14],recreat:1,recurs:1,red:1,redhat:3,redi:3,redirect:0,redistribut:7,redund:14,referenc:[2,14],reflect:[2,5,8,14],refrain:2,refresh:1,refresh_from_db:1,regard:[1,7],regardless:[0,1],regarless:1,regex:[0,1],regex_kei:1,regexp:0,regist:[8,9,12],regular:[1,11],reinit_frequ:12,rel:[1,11,14],relat:[1,2,3,4,6],relationship:2,relativedelta:1,relev:4,reli:[1,12],reliabl:1,remain:[7,12],rememb:[1,14],remot:[1,3,11],remote_base_log_fold:3,remote_full_path:1,remote_host:1,remote_log_conn_id:3,remote_port:1,remov:1,render:[0,1,2,8,14],render_templ:1,render_template_from_field:1,renew:[0,12],repeat:2,repl:12,replac:[1,7,12],report:[0,1],repositori:[3,14],repres:[1,2,4,7,11],represent:[1,15],reproduc:7,reproduct:7,request:[0,1,2],requir:[1,2,3,4,5,6,7,12],resembl:2,reset:0,resetdb:0,resid:2,resolv:1,resourc:[],respect:[0,1,2,14],respons:[1,7],response_check:1,rest:1,restart:[2,3,4],restrict:12,restructuredtext:2,result:[1,2,7,9,14],resum:0,
 retain:7,retri:[1,2,14],retriev:[1,2],retrieve_fil:1,retry_delai:[1,14],return_valu:1,reus:8,review:5,revis:7,riccomini:10,rich:[0,2,5,15],rid:1,right:[1,2,7,9],rightmost:2,risk:7,roadmap:[],role:3,roll:[],room:1,room_id:1,root:[1,2],root_dag:1,round:[1,4],rout:12,routin:[1,2],row:[1,2],royalti:7,rule:[],run:[],run_a:1,run_and_check:1,run_as_own:[1,12],run_cli:1,run_id:[0,1,11],runme_0:13,runnabl:[1,2],rwc:12,s3_conn_id:1,s3_default:1,s3_kei:1,s3connect:1,s3distcp:1,s3filetransferoper:2,s3hook:[1,3],s3keysensor:[1,6],s3prefixsensor:6,s3tohivetransf:1,safe_mod:1,sai:[1,2,4,11],said:2,samaccountnam:12,samba:[1,6],samba_conn_id:1,samba_default:1,same:[1,2,3,7,9,12],sampl:[1,3],sample_s:1,sandbox:3,save:[3,4,9,14],scala:1,scalabl:5,scan:[1,2],scene:[6,11],schedul:[],schedule_interv:[1,2,4,11,14],scheduler_interv:4,scheduler_run:3,schema:[1,2,3],scratch:14,screen:14,screenshot:[],script:[],script_begin_tag:1,script_loc:1,search:[1,2],search_path:3,search_scop:12,second:[1,2,4,14],secret:
 [12,15],section:[1,2,3,4,7,12,13,14],secur:[],see:[1,2,3,4,7,11,12,13,14,15],seen:[3,15],segment:1,select:[1,2,9,11,12],self:[1,8],sell:7,semicolon:1,send:[0,1,2,10],send_room_notif:1,sender:1,sensor:[1,2,4],sensor_queu:2,sent:[1,2,3,7],separ:[1,2,7,14],sequenti:[1,11,13],sequentialexecutor:[1,2,13],serial:[0,1],serv:[0,3,14],serve_log:[0,3],server:[0,1,3,8,12,13,14],servic:[0,1,2,3,6,7,11,12],session:[1,12],set:[],set_depend:[1,14],set_downstream:[1,2,14],set_the_password:12,set_upstream:[1,2,14],setgid:3,setuid:3,setup:[3,12],sever:2,shall:7,shape:11,share:[1,2,3,7,8,9],shell:12,shellcmd:1,ship:[0,1],ship_dag:0,shop:7,shortcircuitoper:1,shortcut:[3,15],shortest:1,should:[1,2,3,4,7,8,11,13,14],shouldn:[1,2],show:[0,1,2,4,8,15],shown:1,sid:10,siddharth:10,side:[1,3],signal:1,similar:[2,3,5],simpl:[1,2,8,9,14],simplehttpoper:1,simpler:14,simplest:12,simpli:[1,2,8,11,12,14],simplifi:1,simul:14,simultan:2,sinc:[1,3,13,14],sing:1,singl:[0,1,2,14],site:12,size:[1,2],skew:8,skip:[0,1,2],s
 la:[],sla_miss_callback:1,slack:[1,6],slack_oper:1,slackapioper:1,slackapipostoper:[1,6],slackoper:2,slave:3,sleep:14,slightli:5,slot:[1,2],slowli:5,slug:12,small:[1,2],smaller:1,smallish:1,snap:5,sock:1,soft_fail:1,softwar:[5,7,10],sole:7,solut:5,some:[1,2,3,4,11,14,15],somehow:3,someth:[2,12,14],sometim:[1,2,14],somewhat:14,somewher:13,soon:[1,2,3,4,11],sort:2,sound:[2,14],sourc:[1,2,7,9,10,15],source_conn_id:1,source_project_dataset_t:1,space:5,span:15,spark:[1,2,5],sparkcmd:1,spawn:3,special:[1,2,4,7],specif:[0,1,2,3,4,7,11,14,15],specifi:[0,1,2,3,4,5,11,12,14],specific_valu:1,spent:15,split:1,split_column:1,sponsor:5,spring:10,sql:[1,2,3,6,9],sql_alchemy_conn:3,sql_alchemy_conn_cmd:3,sqlachemi:1,sqlalchemi:[1,2,3],sqlite:[1,13],sqlitehook:1,sqloper:2,sqlsensor:1,squeez:1,sscursor:1,ssh:[1,12],ssh_default:1,ssh_hook:1,sshexecuteoper:1,sshhook:1,sshpass:1,ssl:1,sslcert:1,sslmode:1,stabil:5,stabl:6,stack:[6,8],stage:[1,2],stai:11,stamp:11,stand:2,standalon:2,standard:[1,2],stanza:
 3,start:[],start_dat:[0,1,2,4,11,14],stat:1,state:[0,1,2,4,7,11,14],state_d:1,statement:[1,2,7],static_babynam:1,static_babynames_partit:1,static_fold:8,static_path:1,static_url_path:8,statu:[0,1,2,3,4,5,7,11,13,14,15],stderr:0,stdout:[0,1,14],step:[1,3,14,15],steven:10,still:[1,4],sting:1,stop:[1,3],stoppag:7,storag:[1,3],store:[0,1,2,3,8,12,13],store_fil:1,store_to_xcom_kei:1,storm:5,str:[1,8,11],straightforward:13,stream:[1,5],string:[0,1,2,3,14],string_data:1,structur:[1,5,14],style:1,sub:[0,1],sub_command:1,sub_commnad:1,sub_dag:[1,2],sub_part:1,subcommand:[1,3,11,14],subdag:[],subdag_oper:2,subdagoper:2,subdir:0,subdirectori:2,subject:[1,7],sublicens:7,submiss:[7,11],submit:[1,7],subpackag:[6,12],subpartit:1,subprocess:[1,11],subscrib:[2,10],subsect:[0,11],subsequ:[0,7,11],subset:1,subtl:2,subtract:1,subtre:12,succe:[1,2],succeed:[0,1,2,4,15],success:[0,1,2,4,5,11,14],successfulli:[1,2,14],suit:5,sum:2,summit:10,sundai:11,super_us:12,supersed:7,superus:12,superuser_filt:12,sup
 pli:[1,3,12],support:[0,1,3,6,7,12],sure:[1,2,3,4,12,14],surfac:14,surgeri:5,sync:[0,1,3,11],sync_to_db:1,synchron:[1,3],syntax:7,sysconfig:3,system:[1,2,3,7],syvineckruyk:10,tab:[1,12],tabl:[1,2,8],table_exist:1,table_nam:[1,2],tablenam:1,tag:[1,4],take:[1,2,3,10,12,13,14,15],talk:[1,10,14],tardif:10,target:[1,3,11],target_field:1,target_partition_s:1,target_tim:1,task:[],task_id:[0,1,2,11,14],task_inst:[1,2],task_instance_key_str:1,task_or_task_list:1,task_param:0,task_regex:[0,1],task_stat:0,taskinst:[1,2],tbl:1,team:[1,12],tell:[0,1],templat:[],template_fold:8,template_searchpath:[1,14],templated_command:14,templates_dict:1,templates_ext:1,temporari:[1,2],tenanc:[],tenant:12,term:7,termin:[2,7],test:[],test_env:2,test_hql:1,test_mod:1,test_plugin:8,testabl:5,testview:8,text:[1,3,7,15],textfil:1,than:[1,2,5,14],thei:[0,1,2,3,5,6,8,9,11,14],them:[0,1,2,8,11,14],themselv:2,theoret:1,theori:[4,7],thereof:7,thi:[0,1,2,3,4,5,7,8,11,12,13,14,15],thin:1,thing:[1,2,4,14],think:[5,14],thi
 rd:7,this_dag_will_be_discov:2,those:[1,2,3,7,12],though:[1,2,5],thousand:2,thread:1,three:[2,3],threshold:8,thrift:1,through:[1,2,6,7,11,13,14],throughout:1,thu:1,ticket:[0,12],tighten:12,time:[1,2,3,4,8,11,12,14,15],timedelta:[1,2,4,11,14],timedeltasensor:[1,4],timefram:1,timeout:[0,1,2],timesensor:1,timestamp:[1,2,11],tip:[2,10],titl:[2,7],tls_ca_cert:1,tls_client_cert:1,tls_client_kei:1,tls_hostnam:1,tls_ssl_version:1,tmp:[1,2],tmp_dir:1,tmp_file:1,todai:1,togeth:[1,2],token:1,toler:[1,2],tomorrow:1,tomorrow_d:1,tomorrow_ds_nodash:1,too:[1,2,11],tool:[1,2,6,8],toolbox:8,top:[],topic:3,tornado:0,tort:7,tour:[13,14],track:[2,7,14],tracker:1,trade:7,trademark:7,tradit:1,tradition:2,train:7,transact:1,transfer:[1,2,7],transform:7,translat:[1,7],transpar:15,travers:1,treat:[1,2],tree:[],tree_view:1,tri:[1,12],trick:10,trigger:[],trigger_dag:[0,11],trigger_dag_id:1,trigger_rul:[1,2],triggerdagrunoper:[1,10],triggerrul:1,troubl:6,troubleshoot:[5,15],truncat:1,truth:1,ts_dim:1,ts_nodash
 :1,tsv:1,tty:1,tunnel:[1,12],tupl:1,turn:[1,2,12],tutori:[],twice:1,two:[1,2,14],txt:1,type:[0,1,2,4,7,9,12],typic:[1,2],ubuntu:3,udf_config:1,uid:12,ultim:[1,14],unarchiv:1,unassign:2,unavail:3,unblock:11,undefin:12,under:[1,2,3,7,8,10],undergo:5,underli:8,underscor:3,understand:[1,4,8,14,15],undesir:11,unexpect:2,unexpectedli:2,union:7,uniqu:[1,14],unit:[3,5,12],unittest:1,unix:1,unless:[1,4,7],unpack:[1,2],unpaus:0,unset:1,unsound:[1,2],unsuccess:1,unsupport:1,until:[1,2,3,5],up_for_retri:1,updat:[1,2,4,12],updateonli:1,upgrad:0,upgradedb:0,upload:1,upon:[2,3,13],upstream:[0,1,2,4,11],upstream_fail:[1,2],upstream_list:1,upstream_task_id:1,uri:[1,2,3,12],url:[1,2,3,8,9,12],usag:[0,1,14],use_beelin:[1,12],user:[0,1,2,3,5,6,12,14],user_defined_macro:1,user_filt:12,user_name_attr:12,user_program_argu:1,usernam:[1,3,12],usr:3,usual:[2,3],utf8:1,utf:1,util:[1,5,13],uuid:1,val:[0,1],valid:[],valu:[0,1,2,3,4,12,14,15],valuecheckoper:1,variabl:[],vast:1,verbal:7,verbos:[1,14],veri:[0,1,2,
 4,10,13,14],verifi:[2,4],version:[0,1,2,5,6,7],vertia:1,vertica:[1,6],vertica_conn_id:1,vertica_default:1,verticahook:1,verticaoper:1,verticaql:1,verticatohivetransf:1,via:[1,2,12],video:[1,10],view:[],virtual:1,virtualenv:2,visibl:[1,2],visual:[5,8,9,14,15],vocabulari:2,volum:1,wai:[1,2,3,4,6,8,11,12,14,15],wait:[0,1,2],wait_for_downstream:[1,4],wait_for_empty_queu:2,walk:[1,14],want:[1,2,3,4,8,11,12,14],warranti:7,wasn:[1,4],wasnt:[],watch:[1,3,4],weapon:9,web:[],webhdfs_conn_id:1,webhdfs_default:1,webhdfshook:1,webhdfssensor:1,webserv:[0,3,4,12,13,14],week:11,weekli:11,weight:1,well:[1,2,3,4,11],were:1,what:[],whatev:[1,2,3,6],when:[1,2,3,4,5,8,9,11,12,14],whenev:1,where:[1,2,3,4,7,11,12,14,15],where_claus:1,wherev:[1,7],whether:[1,7,11],which:[0,1,2,3,4,7,11,12,13,14],whitelist:12,who:8,whole:[1,2,7],whom:7,whose:11,why:[],wiedmer:10,wiki:10,wildcard:1,wildcard_kei:1,wildcard_match:1,win:1,window:11,winter:10,wire:2,within:[0,1,2,3,4,7],without:[0,1,2,7,11,12],won:[1,4,6],word:[
 2,3,11,15],work:[0,1,2,3,4,5,7,8,9,11,12,13],worker:[0,1,2,3,4,5,6,8,12,14],worker_timeout:0,workerclass:0,workflow:[],workload:8,worldwid:7,worri:14,worth:1,would:[1,2,3,4,5,11,12,14],wouldn:[1,4],wrap:[1,14],wrapper:1,write:[3,5,7,8,9,14],write_disposit:1,write_empti:1,written:[1,7,14],wrong:14,www:[1,2,7],x01:1,xcom:[],xcom_al:1,xcom_pul:[1,2],xcom_push:[1,2],xcom_return_kei:1,xml:12,xst:12,xxx:1,yaml:2,year:[1,11],yearli:11,yellow:1,yesterdai:1,yesterday_d:1,yesterday_ds_nodash:1,yet:[1,2,5,12,15],you:[1,2,3,4,5,6,7,8,9,11,12,13,14,15],your:[],yourself:1,youtub:1,yum:6,yvinec:10,yyyi:[0,1,2],yyyymmdd:1,zero:1,zip:[1,2],zip_dag:2,zip_dag_cont:2,zoom:2},titles:["Command Line Interface","API Reference","Concepts","Configuration","FAQ","Apache Airflow (incubating) Documentation","Installation","License","Plugins","Data Profiling","Project","Scheduling & Triggers","Security","Quick Start","Tutorial","UI / Screenshots"],titleterms:{"default":[1,2,14],"function":2,"import":14,addit
 :2,adhoc:9,airflow:[5,6,8],apach:5,api:1,argument:[2,14],assign:2,authent:12,backend:3,backfil:14,baseoper:1,basesensoroper:1,beyond:5,bitshift:2,branch:2,build:8,celeri:3,chart:[9,15],cluster:2,code:15,command:[0,14],committ:10,commun:[1,3],composit:2,concept:2,configur:3,connect:[2,3],content:5,context:[2,15],contribut:[1,3],core:2,dag:[2,11,14,15],data:9,definit:14,depend:14,disclaim:[],document:[2,5],durat:15,enabl:12,enterpris:12,exampl:[8,14],executor:1,extern:11,extra:6,faq:4,file:14,form:9,gantt:15,get:6,ghe:12,github:12,graph:15,histori:10,hook:[1,2],horizon:5,idea:2,incub:5,instal:6,instanc:[2,15],instanti:14,integr:3,interfac:[0,8],jinja:[2,14],keep:11,kerbero:12,ldap:12,licens:7,limit:12,line:[0,14],link:10,log:3,macro:1,manag:2,menu:15,meso:3,metadata:14,mind:11,model:1,modul:14,multi:12,next:[13,14],note:2,oper:[1,2],option:3,out:3,own:12,packag:[2,6],password:12,pipelin:14,plugin:8,polici:2,pool:2,principl:5,profil:9,project:10,queri:9,queue:2,quick:13,recap:14,refer:
 1,resourc:10,roadmap:10,roll:12,rule:2,run:[11,14],scale:3,schedul:11,scope:2,screenshot:[9,15],script:14,secur:12,set:[3,12,14],sla:2,start:13,subdag:2,systemd:3,task:[2,14,15],templat:[2,14],tenanc:12,test:14,top:8,tree:15,trigger:[2,11],tutori:14,undead:2,upstart:3,valid:14,variabl:[1,2,15],view:15,web:12,what:[8,13,14],why:8,workflow:2,xcom:2,your:12,zombi:2}})
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/tutorial.html
----------------------------------------------------------------------
diff --git a/tutorial.html b/tutorial.html
index 2b379c9..abcbbc0 100644
--- a/tutorial.html
+++ b/tutorial.html
@@ -169,7 +169,7 @@ complicated, a line by line explanation follows below.</p>
 <span class="sd">https://github.com/airbnb/airflow/blob/master/airflow/example_dags/tutorial.py</span>
 <span class="sd">&quot;&quot;&quot;</span>
 <span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">DAG</span>
-<span class="kn">from</span> <span class="nn">airflow.operators</span> <span class="k">import</span> <span class="n">BashOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.operators.bash_operator</span> <span class="k">import</span> <span class="n">BashOperator</span>
 <span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span>
 
 
@@ -245,7 +245,7 @@ Airflow DAG object. Let&#8217;s start by importing the libraries we will need.</
 <span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">DAG</span>
 
 <span class="c1"># Operators; we need this to operate!</span>
-<span class="kn">from</span> <span class="nn">airflow.operators</span> <span class="k">import</span> <span class="n">BashOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.operators.bash_operator</span> <span class="k">import</span> <span class="n">BashOperator</span>
 </pre></div>
 </div>
 </div>
@@ -397,7 +397,7 @@ something like this:</p>
 <span class="sd">http://airflow.readthedocs.org/en/latest/tutorial.html</span>
 <span class="sd">&quot;&quot;&quot;</span>
 <span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">DAG</span>
-<span class="kn">from</span> <span class="nn">airflow.operators</span> <span class="k">import</span> <span class="n">BashOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.operators.bash_operator</span> <span class="k">import</span> <span class="n">BashOperator</span>
 <span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span>
 
 

[3/5] incubator-airflow-site git commit: Updating the disclaimer

[ma...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/hive_to_druid.html
----------------------------------------------------------------------
diff --git a/_modules/hive_to_druid.html b/_modules/hive_to_druid.html
index cbd1586..d85bfb3 100644
--- a/_modules/hive_to_druid.html
+++ b/_modules/hive_to_druid.html
@@ -137,9 +137,24 @@
            <div itemprop="articleBody">
             
   <h1>Source code for hive_to_druid</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
-
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span><span class="p">,</span> <span class="n">DruidHook</span><span class="p">,</span> <span class="n">HiveMetastoreHook</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span><span class="p">,</span> <span class="n">HiveMetastoreHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.druid_hook</span> <span class="kn">import</span> <span class="n">DruidHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 
@@ -240,19 +255,20 @@
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Inserting rows into Druid&quot;</span><span class="p">)</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;HDFS path: &quot;</span> <span class="o">+</span> <span class="n">static_path</span><span class="p">)</span>
 
-        <span class="n">druid</span><span class="o">.</span><span class="n">load_from_hdfs</span><span class="p">(</span>
-            <span class="n">datasource</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">druid_datasource</span><span class="p">,</span>
-            <span class="n">intervals</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">intervals</span><span class="p">,</span>
-            <span class="n">static_path</span><span class="o">=</span><span class="n">static_path</span><span class="p">,</span> <span class="n">ts_dim</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">ts_dim</span><span class="p">,</span>
-            <span class="n">columns</span><span class="o">=</span><span class="n">columns</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">num_shards</span><span class="p">,</span> <span class="n">target_partition_size</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">target_partition_size</span><span class="p">,</span>
-            <span class="n">metric_spec</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">metric_spec</span><span class="p">,</span> <span class="n">hadoop_dependency_coordinates</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">hadoop_dependency_coordinates</span><span class="p">)</span>
-        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Load seems to have succeeded!&quot;</span><span class="p">)</span>
-
-        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
-            <span class="s2">&quot;Cleaning up by dropping the temp &quot;</span>
-            <span class="s2">&quot;Hive table {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">hive_table</span><span class="p">))</span>
-        <span class="n">hql</span> <span class="o">=</span> <span class="s2">&quot;DROP TABLE IF EXISTS {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">hive_table</span><span class="p">)</span>
-        <span class="n">hive</span><span class="o">.</span><span class="n">run_cli</span><span class="p">(</span><span class="n">hql</span><span class="p">)</span></div>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">druid</span><span class="o">.</span><span class="n">load_from_hdfs</span><span class="p">(</span>
+                <span class="n">datasource</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">druid_datasource</span><span class="p">,</span>
+                <span class="n">intervals</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">intervals</span><span class="p">,</span>
+                <span class="n">static_path</span><span class="o">=</span><span class="n">static_path</span><span class="p">,</span> <span class="n">ts_dim</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">ts_dim</span><span class="p">,</span>
+                <span class="n">columns</span><span class="o">=</span><span class="n">columns</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">num_shards</span><span class="p">,</span> <span class="n">target_partition_size</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">target_partition_size</span><span class="p">,</span>
+                <span class="n">metric_spec</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">metric_spec</span><span class="p">,</span> <span class="n">hadoop_dependency_coordinates</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">hadoop_dependency_coordinates</span><span class="p">)</span>
+            <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Load seems to have succeeded!&quot;</span><span class="p">)</span>
+        <span class="k">finally</span><span class="p">:</span>
+            <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+                <span class="s2">&quot;Cleaning up by dropping the temp &quot;</span>
+                <span class="s2">&quot;Hive table {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">hive_table</span><span class="p">))</span>
+            <span class="n">hql</span> <span class="o">=</span> <span class="s2">&quot;DROP TABLE IF EXISTS {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">hive_table</span><span class="p">)</span>
+            <span class="n">hive</span><span class="o">.</span><span class="n">run_cli</span><span class="p">(</span><span class="n">hql</span><span class="p">)</span></div>
 </pre></div>
 
            </div>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/hive_to_mysql.html
----------------------------------------------------------------------
diff --git a/_modules/hive_to_mysql.html b/_modules/hive_to_mysql.html
index 6a5bedd..867023b 100644
--- a/_modules/hive_to_mysql.html
+++ b/_modules/hive_to_mysql.html
@@ -137,9 +137,24 @@
            <div itemprop="articleBody">
             
   <h1>Source code for hive_to_mysql</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
-
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HiveServer2Hook</span><span class="p">,</span> <span class="n">MySqlHook</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveServer2Hook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.mysql_hook</span> <span class="kn">import</span> <span class="n">MySqlHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/hive_to_samba_operator.html
----------------------------------------------------------------------
diff --git a/_modules/hive_to_samba_operator.html b/_modules/hive_to_samba_operator.html
index 3008244..a69680d 100644
--- a/_modules/hive_to_samba_operator.html
+++ b/_modules/hive_to_samba_operator.html
@@ -137,10 +137,25 @@
            <div itemprop="articleBody">
             
   <h1>Source code for hive_to_samba_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
 <span class="kn">import</span> <span class="nn">tempfile</span>
 
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HiveServer2Hook</span><span class="p">,</span> <span class="n">SambaHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveServer2Hook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.samba_hook</span> <span class="kn">import</span> <span class="n">SambaHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/http_hook.html
----------------------------------------------------------------------
diff --git a/_modules/http_hook.html b/_modules/http_hook.html
index 98ed5ad..b62b0d0 100644
--- a/_modules/http_hook.html
+++ b/_modules/http_hook.html
@@ -150,7 +150,7 @@
 <span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
 <span class="c1"># See the License for the specific language governing permissions and</span>
 <span class="c1"># limitations under the License.</span>
-<span class="c1">#</span>
+
 <span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
 <span class="kn">import</span> <span class="nn">logging</span>
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/http_operator.html
----------------------------------------------------------------------
diff --git a/_modules/http_operator.html b/_modules/http_operator.html
index 942ae0e..00496c9 100644
--- a/_modules/http_operator.html
+++ b/_modules/http_operator.html
@@ -137,10 +137,24 @@
            <div itemprop="articleBody">
             
   <h1>Source code for http_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
 
 <span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span>
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HttpHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.http_hook</span> <span class="kn">import</span> <span class="n">HttpHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/mssql_hook.html
----------------------------------------------------------------------
diff --git a/_modules/mssql_hook.html b/_modules/mssql_hook.html
index b3b5503..9aff59d 100644
--- a/_modules/mssql_hook.html
+++ b/_modules/mssql_hook.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for mssql_hook</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">pymssql</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">pymssql</span>
 
 <span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="kn">import</span> <span class="n">DbApiHook</span>
 
@@ -157,10 +171,11 @@
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">mssql_conn_id</span><span class="p">)</span>
         <span class="n">conn</span> <span class="o">=</span> <span class="n">pymssql</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span>
-            <span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="p">,</span>
-            <span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">,</span>
-            <span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">,</span>
-            <span class="n">conn</span><span class="o">.</span><span class="n">schema</span><span class="p">)</span>
+            <span class="n">server</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="p">,</span>
+            <span class="n">user</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">,</span>
+            <span class="n">password</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">,</span>
+            <span class="n">database</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span>
+            <span class="n">port</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">)</span>
         <span class="k">return</span> <span class="n">conn</span></div>
 
     <span class="k">def</span> <span class="nf">set_autocommit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">conn</span><span class="p">,</span> <span class="n">autocommit</span><span class="p">):</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/mssql_operator.html
----------------------------------------------------------------------
diff --git a/_modules/mssql_operator.html b/_modules/mssql_operator.html
index dcc2872..e933004 100644
--- a/_modules/mssql_operator.html
+++ b/_modules/mssql_operator.html
@@ -137,9 +137,23 @@
            <div itemprop="articleBody">
             
   <h1>Source code for mssql_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
-
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">MsSqlHook</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.mssql_hook</span> <span class="kn">import</span> <span class="n">MsSqlHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/mssql_to_hive.html
----------------------------------------------------------------------
diff --git a/_modules/mssql_to_hive.html b/_modules/mssql_to_hive.html
index 451dc7f..590dfb8 100644
--- a/_modules/mssql_to_hive.html
+++ b/_modules/mssql_to_hive.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for mssql_to_hive</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">chr</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">chr</span>
 <span class="kn">from</span> <span class="nn">collections</span> <span class="kn">import</span> <span class="n">OrderedDict</span>
 <span class="kn">import</span> <span class="nn">unicodecsv</span> <span class="kn">as</span> <span class="nn">csv</span>
 <span class="kn">import</span> <span class="nn">logging</span>
@@ -145,7 +159,8 @@
 <span class="kn">import</span> <span class="nn">pymssql</span>
 
 
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span><span class="p">,</span> <span class="n">MsSqlHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.mssql_hook</span> <span class="kn">import</span> <span class="n">MsSqlHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/mysql_hook.html
----------------------------------------------------------------------
diff --git a/_modules/mysql_hook.html b/_modules/mysql_hook.html
index 23d3726..941737f 100644
--- a/_modules/mysql_hook.html
+++ b/_modules/mysql_hook.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for mysql_hook</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">MySQLdb</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">MySQLdb</span>
 <span class="kn">import</span> <span class="nn">MySQLdb.cursors</span>
 
 <span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="kn">import</span> <span class="n">DbApiHook</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/mysql_operator.html
----------------------------------------------------------------------
diff --git a/_modules/mysql_operator.html b/_modules/mysql_operator.html
index 331f2b4..a1fc11a 100644
--- a/_modules/mysql_operator.html
+++ b/_modules/mysql_operator.html
@@ -137,9 +137,23 @@
            <div itemprop="articleBody">
             
   <h1>Source code for mysql_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
-
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">MySqlHook</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.mysql_hook</span> <span class="kn">import</span> <span class="n">MySqlHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/mysql_to_hive.html
----------------------------------------------------------------------
diff --git a/_modules/mysql_to_hive.html b/_modules/mysql_to_hive.html
index fa6b20f..cc8496b 100644
--- a/_modules/mysql_to_hive.html
+++ b/_modules/mysql_to_hive.html
@@ -137,14 +137,29 @@
            <div itemprop="articleBody">
             
   <h1>Source code for mysql_to_hive</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">chr</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">chr</span>
 <span class="kn">from</span> <span class="nn">collections</span> <span class="kn">import</span> <span class="n">OrderedDict</span>
 <span class="kn">import</span> <span class="nn">unicodecsv</span> <span class="kn">as</span> <span class="nn">csv</span>
 <span class="kn">import</span> <span class="nn">logging</span>
 <span class="kn">from</span> <span class="nn">tempfile</span> <span class="kn">import</span> <span class="n">NamedTemporaryFile</span>
 <span class="kn">import</span> <span class="nn">MySQLdb</span>
 
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span><span class="p">,</span> <span class="n">MySqlHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.mysql_hook</span> <span class="kn">import</span> <span class="n">MySqlHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/postgres_hook.html
----------------------------------------------------------------------
diff --git a/_modules/postgres_hook.html b/_modules/postgres_hook.html
index d8896ac..3ecfa69 100644
--- a/_modules/postgres_hook.html
+++ b/_modules/postgres_hook.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for postgres_hook</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">psycopg2</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">psycopg2</span>
 <span class="kn">import</span> <span class="nn">psycopg2.extensions</span>
 
 <span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="kn">import</span> <span class="n">DbApiHook</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/postgres_operator.html
----------------------------------------------------------------------
diff --git a/_modules/postgres_operator.html b/_modules/postgres_operator.html
index 4eb0e00..5ade37d 100644
--- a/_modules/postgres_operator.html
+++ b/_modules/postgres_operator.html
@@ -137,9 +137,23 @@
            <div itemprop="articleBody">
             
   <h1>Source code for postgres_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
-
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">PostgresHook</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.postgres_hook</span> <span class="kn">import</span> <span class="n">PostgresHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/presto_check_operator.html
----------------------------------------------------------------------
diff --git a/_modules/presto_check_operator.html b/_modules/presto_check_operator.html
index 52dc0e6..e24b271 100644
--- a/_modules/presto_check_operator.html
+++ b/_modules/presto_check_operator.html
@@ -137,8 +137,22 @@
            <div itemprop="articleBody">
             
   <h1>Source code for presto_check_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">PrestoHook</span>
-<span class="kn">from</span> <span class="nn">airflow.operators</span> <span class="kn">import</span> <span class="n">CheckOperator</span><span class="p">,</span> <span class="n">ValueCheckOperator</span><span class="p">,</span> <span class="n">IntervalCheckOperator</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.presto_hook</span> <span class="kn">import</span> <span class="n">PrestoHook</span>
+<span class="kn">from</span> <span class="nn">airflow.operators.check_operator</span> <span class="kn">import</span> <span class="n">CheckOperator</span><span class="p">,</span> <span class="n">ValueCheckOperator</span><span class="p">,</span> <span class="n">IntervalCheckOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/presto_hook.html
----------------------------------------------------------------------
diff --git a/_modules/presto_hook.html b/_modules/presto_hook.html
index 73e4cc0..49b93a9 100644
--- a/_modules/presto_hook.html
+++ b/_modules/presto_hook.html
@@ -137,13 +137,28 @@
            <div itemprop="articleBody">
             
   <h1>Source code for presto_hook</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
+<span class="kn">import</span> <span class="nn">logging</span>
+
 <span class="kn">from</span> <span class="nn">pyhive</span> <span class="kn">import</span> <span class="n">presto</span>
 <span class="kn">from</span> <span class="nn">pyhive.exc</span> <span class="kn">import</span> <span class="n">DatabaseError</span>
 
 <span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="kn">import</span> <span class="n">DbApiHook</span>
 
-<span class="kn">import</span> <span class="nn">logging</span>
 <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">&quot;pyhive&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">INFO</span><span class="p">)</span>
 
 
@@ -204,8 +219,7 @@
             <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">PrestoHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">get_first</span><span class="p">(</span>
                 <span class="bp">self</span><span class="o">.</span><span class="n">_strip_sql</span><span class="p">(</span><span class="n">hql</span><span class="p">),</span> <span class="n">parameters</span><span class="p">)</span>
         <span class="k">except</span> <span class="n">DatabaseError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
-            <span class="n">obj</span> <span class="o">=</span> <span class="nb">eval</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
-            <span class="k">raise</span> <span class="n">PrestoException</span><span class="p">(</span><span class="n">obj</span><span class="p">[</span><span class="s1">&#39;message&#39;</span><span class="p">])</span></div>
+            <span class="k">raise</span> <span class="n">PrestoException</span><span class="p">(</span><span class="n">e</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="s1">&#39;message&#39;</span><span class="p">])</span></div>
 
 <div class="viewcode-block" id="PrestoHook.get_pandas_df"><a class="viewcode-back" href="../code.html#airflow.hooks.PrestoHook.get_pandas_df">[docs]</a>    <span class="k">def</span> <span class="nf">get_pandas_df</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">hql</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
@@ -217,8 +231,7 @@
             <span class="n">cursor</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_strip_sql</span><span class="p">(</span><span class="n">hql</span><span class="p">),</span> <span class="n">parameters</span><span class="p">)</span>
             <span class="n">data</span> <span class="o">=</span> <span class="n">cursor</span><span class="o">.</span><span class="n">fetchall</span><span class="p">()</span>
         <span class="k">except</span> <span class="n">DatabaseError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
-            <span class="n">obj</span> <span class="o">=</span> <span class="nb">eval</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
-            <span class="k">raise</span> <span class="n">PrestoException</span><span class="p">(</span><span class="n">obj</span><span class="p">[</span><span class="s1">&#39;message&#39;</span><span class="p">])</span>
+            <span class="k">raise</span> <span class="n">PrestoException</span><span class="p">(</span><span class="n">e</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="s1">&#39;message&#39;</span><span class="p">])</span>
         <span class="n">column_descriptions</span> <span class="o">=</span> <span class="n">cursor</span><span class="o">.</span><span class="n">description</span>
         <span class="k">if</span> <span class="n">data</span><span class="p">:</span>
             <span class="n">df</span> <span class="o">=</span> <span class="n">pandas</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/python_operator.html
----------------------------------------------------------------------
diff --git a/_modules/python_operator.html b/_modules/python_operator.html
index 0f0de31..059ebb2 100644
--- a/_modules/python_operator.html
+++ b/_modules/python_operator.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for python_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
 <span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">datetime</span>
 <span class="kn">import</span> <span class="nn">logging</span>
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/qubole_operator.html
----------------------------------------------------------------------
diff --git a/_modules/qubole_operator.html b/_modules/qubole_operator.html
index 07ea626..fb33047 100644
--- a/_modules/qubole_operator.html
+++ b/_modules/qubole_operator.html
@@ -137,9 +137,23 @@
            <div itemprop="articleBody">
             
   <h1>Source code for qubole_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
-<span class="kn">from</span> <span class="nn">airflow.contrib.hooks</span> <span class="kn">import</span> <span class="n">QuboleHook</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.qubole_hook</span> <span class="kn">import</span> <span class="n">QuboleHook</span>
 
 
 <div class="viewcode-block" id="QuboleOperator"><a class="viewcode-back" href="../code.html#airflow.contrib.operators.QuboleOperator">[docs]</a><span class="k">class</span> <span class="nc">QuboleOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
@@ -220,7 +234,7 @@
 
     <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;query&#39;</span><span class="p">,</span> <span class="s1">&#39;script_location&#39;</span><span class="p">,</span> <span class="s1">&#39;sub_command&#39;</span><span class="p">,</span> <span class="s1">&#39;script&#39;</span><span class="p">,</span> <span class="s1">&#39;files&#39;</span><span class="p">,</span> <span class="s1">&#39;archives&#39;</span><span class="p">,</span> <span class="s1">&#39;program&#39;</span><span class="p">,</span> <span class="s1">&#39;cmdline&#39;</span><span class="p">,</span>
                        <span class="s1">&#39;sql&#39;</span><span class="p">,</span> <span class="s1">&#39;where_clause&#39;</span><span class="p">,</span> <span class="s1">&#39;extract_query&#39;</span><span class="p">,</span> <span class="s1">&#39;boundary_query&#39;</span><span class="p">,</span> <span class="s1">&#39;macros&#39;</span><span class="p">,</span> <span class="s1">&#39;tags&#39;</span><span class="p">,</span> <span class="s1">&#39;name&#39;</span><span class="p">,</span> <span class="s1">&#39;parameters&#39;</span><span class="p">)</span>
-    <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;.txt&#39;</span><span class="p">)</span>
+    <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;.txt&#39;</span><span class="p">,)</span>
     <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#3064A1&#39;</span>
     <span class="n">ui_fgcolor</span> <span class="o">=</span> <span class="s1">&#39;#fff&#39;</span>
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/s3_to_hive_operator.html
----------------------------------------------------------------------
diff --git a/_modules/s3_to_hive_operator.html b/_modules/s3_to_hive_operator.html
index dc23eb5..ad7715e 100644
--- a/_modules/s3_to_hive_operator.html
+++ b/_modules/s3_to_hive_operator.html
@@ -137,13 +137,28 @@
            <div itemprop="articleBody">
             
   <h1>Source code for s3_to_hive_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">next</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">next</span>
 <span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">zip</span>
 <span class="kn">import</span> <span class="nn">logging</span>
 <span class="kn">from</span> <span class="nn">tempfile</span> <span class="kn">import</span> <span class="n">NamedTemporaryFile</span>
 
 <span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span>
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span><span class="p">,</span> <span class="n">S3Hook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.S3_hook</span> <span class="kn">import</span> <span class="n">S3Hook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/sensors.html
----------------------------------------------------------------------
diff --git a/_modules/sensors.html b/_modules/sensors.html
index 19336b2..fab6313 100644
--- a/_modules/sensors.html
+++ b/_modules/sensors.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for sensors</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">print_function</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">print_function</span>
 <span class="kn">from</span> <span class="nn">future</span> <span class="kn">import</span> <span class="n">standard_library</span>
 <span class="n">standard_library</span><span class="o">.</span><span class="n">install_aliases</span><span class="p">()</span>
 <span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
@@ -146,10 +160,11 @@
 <span class="kn">from</span> <span class="nn">urllib.parse</span> <span class="kn">import</span> <span class="n">urlparse</span>
 <span class="kn">from</span> <span class="nn">time</span> <span class="kn">import</span> <span class="n">sleep</span>
 
+<span class="kn">import</span> <span class="nn">airflow</span>
 <span class="kn">from</span> <span class="nn">airflow</span> <span class="kn">import</span> <span class="n">hooks</span><span class="p">,</span> <span class="n">settings</span>
 <span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span><span class="p">,</span> <span class="n">AirflowSensorTimeout</span><span class="p">,</span> <span class="n">AirflowSkipException</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span><span class="p">,</span> <span class="n">TaskInstance</span><span class="p">,</span> <span class="n">Connection</span> <span class="k">as</span> <span class="n">DB</span>
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="kn">import</span> <span class="n">BaseHook</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="kn">import</span> <span class="n">State</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 
@@ -193,12 +208,12 @@
     <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
         <span class="n">started_at</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span>
         <span class="k">while</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">poke</span><span class="p">(</span><span class="n">context</span><span class="p">):</span>
-            <span class="n">sleep</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">poke_interval</span><span class="p">)</span>
-            <span class="k">if</span> <span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> <span class="o">-</span> <span class="n">started_at</span><span class="p">)</span><span class="o">.</span><span class="n">seconds</span> <span class="o">&gt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">:</span>
+            <span class="k">if</span> <span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> <span class="o">-</span> <span class="n">started_at</span><span class="p">)</span><span class="o">.</span><span class="n">total_seconds</span><span class="p">()</span> <span class="o">&gt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">:</span>
                 <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">soft_fail</span><span class="p">:</span>
                     <span class="k">raise</span> <span class="n">AirflowSkipException</span><span class="p">(</span><span class="s1">&#39;Snap. Time is OUT.&#39;</span><span class="p">)</span>
                 <span class="k">else</span><span class="p">:</span>
                     <span class="k">raise</span> <span class="n">AirflowSensorTimeout</span><span class="p">(</span><span class="s1">&#39;Snap. Time is OUT.&#39;</span><span class="p">)</span>
+            <span class="n">sleep</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">poke_interval</span><span class="p">)</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Success criteria met. Exiting.&quot;</span><span class="p">)</span>
 
 
@@ -303,8 +318,14 @@
 <span class="sd">    :type allowed_states: list</span>
 <span class="sd">    :param execution_delta: time difference with the previous execution to</span>
 <span class="sd">        look at, the default is the same execution_date as the current task.</span>
-<span class="sd">        For yesterday, use [positive!] datetime.timedelta(days=1)</span>
+<span class="sd">        For yesterday, use [positive!] datetime.timedelta(days=1). Either</span>
+<span class="sd">        execution_delta or execution_date_fn can be passed to</span>
+<span class="sd">        ExternalTaskSensor, but not both.</span>
 <span class="sd">    :type execution_delta: datetime.timedelta</span>
+<span class="sd">    :param execution_date_fn: function that receives the current execution date</span>
+<span class="sd">        and returns the desired execution date to query. Either execution_delta</span>
+<span class="sd">        or execution_date_fn can be passed to ExternalTaskSensor, but not both.</span>
+<span class="sd">    :type execution_date_fn: callable</span>
 <span class="sd">    &quot;&quot;&quot;</span>
 
     <span class="nd">@apply_defaults</span>
@@ -314,16 +335,25 @@
             <span class="n">external_task_id</span><span class="p">,</span>
             <span class="n">allowed_states</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
             <span class="n">execution_delta</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+            <span class="n">execution_date_fn</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
             <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
         <span class="nb">super</span><span class="p">(</span><span class="n">ExternalTaskSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">allowed_states</span> <span class="o">=</span> <span class="n">allowed_states</span> <span class="ow">or</span> <span class="p">[</span><span class="n">State</span><span class="o">.</span><span class="n">SUCCESS</span><span class="p">]</span>
+        <span class="k">if</span> <span class="n">execution_delta</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span> <span class="ow">and</span> <span class="n">execution_date_fn</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                <span class="s1">&#39;Only one of `execution_date` or `execution_date_fn` may&#39;</span>
+                <span class="s1">&#39;be provided to ExternalTaskSensor; not both.&#39;</span><span class="p">)</span>
+
         <span class="bp">self</span><span class="o">.</span><span class="n">execution_delta</span> <span class="o">=</span> <span class="n">execution_delta</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">execution_date_fn</span> <span class="o">=</span> <span class="n">execution_date_fn</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">external_dag_id</span> <span class="o">=</span> <span class="n">external_dag_id</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">external_task_id</span> <span class="o">=</span> <span class="n">external_task_id</span>
 
     <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
         <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_delta</span><span class="p">:</span>
             <span class="n">dttm</span> <span class="o">=</span> <span class="n">context</span><span class="p">[</span><span class="s1">&#39;execution_date&#39;</span><span class="p">]</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_delta</span>
+        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date_fn</span><span class="p">:</span>
+            <span class="n">dttm</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date_fn</span><span class="p">(</span><span class="n">context</span><span class="p">[</span><span class="s1">&#39;execution_date&#39;</span><span class="p">])</span>
         <span class="k">else</span><span class="p">:</span>
             <span class="n">dttm</span> <span class="o">=</span> <span class="n">context</span><span class="p">[</span><span class="s1">&#39;execution_date&#39;</span><span class="p">]</span>
 
@@ -346,15 +376,91 @@
         <span class="k">return</span> <span class="n">count</span></div>
 
 
+<span class="k">class</span> <span class="nc">NamedHivePartitionSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Waits for a set of partitions to show up in Hive.</span>
+
+<span class="sd">    :param partition_names: List of fully qualified names of the</span>
+<span class="sd">        partitions to wait for. A fully qualified name is of the</span>
+<span class="sd">        form schema.table/pk1=pv1/pk2=pv2, for example,</span>
+<span class="sd">        default.users/ds=2016-01-01. This is passed as is to the metastore</span>
+<span class="sd">        Thrift client &quot;get_partitions_by_name&quot; method. Note that</span>
+<span class="sd">        you cannot use logical operators as in HivePartitionSensor.</span>
+<span class="sd">    :type partition_names: list of strings</span>
+<span class="sd">    :param metastore_conn_id: reference to the metastore thrift service</span>
+<span class="sd">        connection id</span>
+<span class="sd">    :type metastore_conn_id: str</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;partition_names&#39;</span><span class="p">,</span> <span class="p">)</span>
+
+    <span class="nd">@apply_defaults</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+            <span class="bp">self</span><span class="p">,</span>
+            <span class="n">partition_names</span><span class="p">,</span>
+            <span class="n">metastore_conn_id</span><span class="o">=</span><span class="s1">&#39;metastore_default&#39;</span><span class="p">,</span>
+            <span class="n">poke_interval</span><span class="o">=</span><span class="mi">60</span><span class="o">*</span><span class="mi">3</span><span class="p">,</span>
+            <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+            <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">NamedHivePartitionSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span>
+            <span class="n">poke_interval</span><span class="o">=</span><span class="n">poke_interval</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">partition_names</span><span class="p">,</span> <span class="nb">basestring</span><span class="p">):</span>
+            <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s1">&#39;partition_names must be an array of strings&#39;</span><span class="p">)</span>
+
+        <span class="k">for</span> <span class="n">partition_name</span> <span class="ow">in</span> <span class="n">partition_names</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">parse_partition_name</span><span class="p">(</span><span class="n">partition_name</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">metastore_conn_id</span> <span class="o">=</span> <span class="n">metastore_conn_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">partition_names</span> <span class="o">=</span> <span class="n">partition_names</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">next_poke_idx</span> <span class="o">=</span> <span class="mi">0</span>
+
+    <span class="k">def</span> <span class="nf">parse_partition_name</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">partition</span><span class="p">):</span>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">schema</span><span class="p">,</span> <span class="n">table_partition</span> <span class="o">=</span> <span class="n">partition</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">)</span>
+            <span class="n">table</span><span class="p">,</span> <span class="n">partition</span> <span class="o">=</span> <span class="n">table_partition</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
+            <span class="k">return</span> <span class="n">schema</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">partition</span>
+        <span class="k">except</span> <span class="ne">ValueError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;Could not parse &#39;</span> <span class="o">+</span> <span class="n">partition</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+
+        <span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s1">&#39;hook&#39;</span><span class="p">):</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">airflow</span><span class="o">.</span><span class="n">hooks</span><span class="o">.</span><span class="n">hive_hooks</span><span class="o">.</span><span class="n">HiveMetastoreHook</span><span class="p">(</span>
+                <span class="n">metastore_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">metastore_conn_id</span><span class="p">)</span>
+
+        <span class="k">def</span> <span class="nf">poke_partition</span><span class="p">(</span><span class="n">partition</span><span class="p">):</span>
+
+            <span class="n">schema</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">partition</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parse_partition_name</span><span class="p">(</span><span class="n">partition</span><span class="p">)</span>
+
+            <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+                <span class="s1">&#39;Poking for {schema}.{table}/{partition}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">())</span>
+            <span class="p">)</span>
+            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">check_for_named_partition</span><span class="p">(</span>
+                <span class="n">schema</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">partition</span><span class="p">)</span>
+
+        <span class="k">while</span> <span class="bp">self</span><span class="o">.</span><span class="n">next_poke_idx</span> <span class="o">&lt;</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">partition_names</span><span class="p">):</span>
+            <span class="k">if</span> <span class="n">poke_partition</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">partition_names</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">next_poke_idx</span><span class="p">]):</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">next_poke_idx</span> <span class="o">+=</span> <span class="mi">1</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="k">return</span> <span class="bp">False</span>
+
+        <span class="k">return</span> <span class="bp">True</span>
+
+
 <div class="viewcode-block" id="HivePartitionSensor"><a class="viewcode-back" href="../code.html#airflow.operators.HivePartitionSensor">[docs]</a><span class="k">class</span> <span class="nc">HivePartitionSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
     <span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">    Waits for a partition to show up in Hive</span>
+<span class="sd">    Waits for a partition to show up in Hive.</span>
+
+<span class="sd">    Note: Because @partition supports general logical operators, it</span>
+<span class="sd">    can be inefficient. Consider using NamedHivePartitionSensor instead if</span>
+<span class="sd">    you don&#39;t need the full flexibility of HivePartitionSensor.</span>
 
 <span class="sd">    :param table: The name of the table to wait for, supports the dot</span>
 <span class="sd">        notation (my_database.my_table)</span>
 <span class="sd">    :type table: string</span>
 <span class="sd">    :param partition: The partition clause to wait for. This is passed as</span>
-<span class="sd">        is to the Metastore Thrift client &quot;get_partitions_by_filter&quot; method,</span>
+<span class="sd">        is to the metastore Thrift client &quot;get_partitions_by_filter&quot; method,</span>
 <span class="sd">        and apparently supports SQL like notation as in `ds=&#39;2015-01-01&#39;</span>
 <span class="sd">        AND type=&#39;value&#39;` and &gt; &lt; sings as in &quot;ds&gt;=2015-01-01&quot;</span>
 <span class="sd">    :type partition: string</span>
@@ -388,7 +494,7 @@
             <span class="s1">&#39;Poking for table {self.schema}.{self.table}, &#39;</span>
             <span class="s1">&#39;partition {self.partition}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
         <span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s1">&#39;hook&#39;</span><span class="p">):</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">HiveMetastoreHook</span><span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">airflow</span><span class="o">.</span><span class="n">hooks</span><span class="o">.</span><span class="n">hive_hooks</span><span class="o">.</span><span class="n">HiveMetastoreHook</span><span class="p">(</span>
                 <span class="n">metastore_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">metastore_conn_id</span><span class="p">)</span>
         <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">check_for_partition</span><span class="p">(</span>
             <span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">table</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">partition</span><span class="p">)</span></div>
@@ -411,7 +517,8 @@
         <span class="bp">self</span><span class="o">.</span><span class="n">hdfs_conn_id</span> <span class="o">=</span> <span class="n">hdfs_conn_id</span>
 
     <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
-        <span class="n">sb</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">HDFSHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">hdfs_conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+        <span class="kn">import</span> <span class="nn">airflow.hooks.hdfs_hook</span>
+        <span class="n">sb</span> <span class="o">=</span> <span class="n">airflow</span><span class="o">.</span><span class="n">hooks</span><span class="o">.</span><span class="n">hdfs_hook</span><span class="o">.</span><span class="n">HDFSHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">hdfs_conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">&quot;snakebite&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">WARNING</span><span class="p">)</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
             <span class="s1">&#39;Poking for file {self.filepath} &#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
@@ -439,7 +546,7 @@
         <span class="bp">self</span><span class="o">.</span><span class="n">webhdfs_conn_id</span> <span class="o">=</span> <span class="n">webhdfs_conn_id</span>
 
     <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
-        <span class="n">c</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">WebHDFSHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">webhdfs_conn_id</span><span class="p">)</span>
+        <span class="n">c</span> <span class="o">=</span> <span class="n">airflow</span><span class="o">.</span><span class="n">hooks</span><span class="o">.</span><span class="n">webhdfs_hook</span><span class="o">.</span><span class="n">WebHDFSHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">webhdfs_conn_id</span><span class="p">)</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
             <span class="s1">&#39;Poking for file {self.filepath} &#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
         <span class="k">return</span> <span class="n">c</span><span class="o">.</span><span class="n">check_for_path</span><span class="p">(</span><span class="n">hdfs_path</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">filepath</span><span class="p">)</span></div>
@@ -495,7 +602,8 @@
         <span class="n">session</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
 
     <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
-        <span class="n">hook</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">S3Hook</span><span class="p">(</span><span class="n">s3_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">s3_conn_id</span><span class="p">)</span>
+        <span class="kn">import</span> <span class="nn">airflow.hooks.S3_hook</span>
+        <span class="n">hook</span> <span class="o">=</span> <span class="n">airflow</span><span class="o">.</span><span class="n">hooks</span><span class="o">.</span><span class="n">S3_hook</span><span class="o">.</span><span class="n">S3Hook</span><span class="p">(</span><span class="n">s3_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">s3_conn_id</span><span class="p">)</span>
         <span class="n">full_url</span> <span class="o">=</span> <span class="s2">&quot;s3://&quot;</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">bucket_name</span> <span class="o">+</span> <span class="s2">&quot;/&quot;</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">bucket_key</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Poking for key : {full_url}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
         <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">wildcard_match</span><span class="p">:</span>
@@ -547,7 +655,8 @@
     <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Poking for prefix : {self.prefix}</span><span class="se">\n</span><span class="s1">&#39;</span>
                      <span class="s1">&#39;in bucket s3://{self.bucket_name}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
-        <span class="n">hook</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">S3Hook</span><span class="p">(</span><span class="n">s3_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">s3_conn_id</span><span class="p">)</span>
+        <span class="kn">import</span> <span class="nn">airflow.hooks.S3_hook</span>
+        <span class="n">hook</span> <span class="o">=</span> <span class="n">airflow</span><span class="o">.</span><span class="n">hooks</span><span class="o">.</span><span class="n">S3_hook</span><span class="o">.</span><span class="n">S3Hook</span><span class="p">(</span><span class="n">s3_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">s3_conn_id</span><span class="p">)</span>
         <span class="k">return</span> <span class="n">hook</span><span class="o">.</span><span class="n">check_for_prefix</span><span class="p">(</span>
             <span class="n">prefix</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">prefix</span><span class="p">,</span>
             <span class="n">delimiter</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delimiter</span><span class="p">,</span>
@@ -639,7 +748,7 @@
         <span class="bp">self</span><span class="o">.</span><span class="n">extra_options</span> <span class="o">=</span> <span class="n">extra_options</span> <span class="ow">or</span> <span class="p">{}</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">response_check</span> <span class="o">=</span> <span class="n">response_check</span>
 
-        <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">HttpHook</span><span class="p">(</span><span class="n">method</span><span class="o">=</span><span class="s1">&#39;GET&#39;</span><span class="p">,</span> <span class="n">http_conn_id</span><span class="o">=</span><span class="n">http_conn_id</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">http_hook</span><span class="o">.</span><span class="n">HttpHook</span><span class="p">(</span><span class="n">method</span><span class="o">=</span><span class="s1">&#39;GET&#39;</span><span class="p">,</span> <span class="n">http_conn_id</span><span class="o">=</span><span class="n">http_conn_id</span><span class="p">)</span>
 
     <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Poking: &#39;</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">endpoint</span><span class="p">)</span>
@@ -652,7 +761,7 @@
                 <span class="c1"># run content check on response</span>
                 <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">response_check</span><span class="p">(</span><span class="n">response</span><span class="p">)</span>
         <span class="k">except</span> <span class="n">AirflowException</span> <span class="k">as</span> <span class="n">ae</span><span class="p">:</span>
-            <span class="k">if</span> <span class="n">ae</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">&quot;404&quot;</span><span class="p">):</span>
+            <span class="k">if</span> <span class="nb">str</span><span class="p">(</span><span class="n">ae</span><span class="p">)</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">&quot;404&quot;</span><span class="p">):</span>
                 <span class="k">return</span> <span class="bp">False</span>
 
             <span class="k">raise</span> <span class="n">ae</span>

[4/5] incubator-airflow-site git commit: Updating the disclaimer

[ma...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/airflow/operators/sensors.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/sensors.html b/_modules/airflow/operators/sensors.html
index cbadb0a..65a0587 100644
--- a/_modules/airflow/operators/sensors.html
+++ b/_modules/airflow/operators/sensors.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for airflow.operators.sensors</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">print_function</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">print_function</span>
 <span class="kn">from</span> <span class="nn">future</span> <span class="kn">import</span> <span class="n">standard_library</span>
 <span class="n">standard_library</span><span class="o">.</span><span class="n">install_aliases</span><span class="p">()</span>
 <span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
@@ -146,10 +160,11 @@
 <span class="kn">from</span> <span class="nn">urllib.parse</span> <span class="kn">import</span> <span class="n">urlparse</span>
 <span class="kn">from</span> <span class="nn">time</span> <span class="kn">import</span> <span class="n">sleep</span>
 
+<span class="kn">import</span> <span class="nn">airflow</span>
 <span class="kn">from</span> <span class="nn">airflow</span> <span class="kn">import</span> <span class="n">hooks</span><span class="p">,</span> <span class="n">settings</span>
 <span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span><span class="p">,</span> <span class="n">AirflowSensorTimeout</span><span class="p">,</span> <span class="n">AirflowSkipException</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span><span class="p">,</span> <span class="n">TaskInstance</span><span class="p">,</span> <span class="n">Connection</span> <span class="k">as</span> <span class="n">DB</span>
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">BaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="kn">import</span> <span class="n">BaseHook</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="kn">import</span> <span class="n">State</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 
@@ -193,12 +208,12 @@
     <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
         <span class="n">started_at</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span>
         <span class="k">while</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">poke</span><span class="p">(</span><span class="n">context</span><span class="p">):</span>
-            <span class="n">sleep</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">poke_interval</span><span class="p">)</span>
-            <span class="k">if</span> <span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> <span class="o">-</span> <span class="n">started_at</span><span class="p">)</span><span class="o">.</span><span class="n">seconds</span> <span class="o">&gt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">:</span>
+            <span class="k">if</span> <span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> <span class="o">-</span> <span class="n">started_at</span><span class="p">)</span><span class="o">.</span><span class="n">total_seconds</span><span class="p">()</span> <span class="o">&gt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">:</span>
                 <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">soft_fail</span><span class="p">:</span>
                     <span class="k">raise</span> <span class="n">AirflowSkipException</span><span class="p">(</span><span class="s1">&#39;Snap. Time is OUT.&#39;</span><span class="p">)</span>
                 <span class="k">else</span><span class="p">:</span>
                     <span class="k">raise</span> <span class="n">AirflowSensorTimeout</span><span class="p">(</span><span class="s1">&#39;Snap. Time is OUT.&#39;</span><span class="p">)</span>
+            <span class="n">sleep</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">poke_interval</span><span class="p">)</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Success criteria met. Exiting.&quot;</span><span class="p">)</span></div>
 
 
@@ -303,8 +318,14 @@
 <span class="sd">    :type allowed_states: list</span>
 <span class="sd">    :param execution_delta: time difference with the previous execution to</span>
 <span class="sd">        look at, the default is the same execution_date as the current task.</span>
-<span class="sd">        For yesterday, use [positive!] datetime.timedelta(days=1)</span>
+<span class="sd">        For yesterday, use [positive!] datetime.timedelta(days=1). Either</span>
+<span class="sd">        execution_delta or execution_date_fn can be passed to</span>
+<span class="sd">        ExternalTaskSensor, but not both.</span>
 <span class="sd">    :type execution_delta: datetime.timedelta</span>
+<span class="sd">    :param execution_date_fn: function that receives the current execution date</span>
+<span class="sd">        and returns the desired execution date to query. Either execution_delta</span>
+<span class="sd">        or execution_date_fn can be passed to ExternalTaskSensor, but not both.</span>
+<span class="sd">    :type execution_date_fn: callable</span>
 <span class="sd">    &quot;&quot;&quot;</span>
 
     <span class="nd">@apply_defaults</span>
@@ -314,16 +335,25 @@
             <span class="n">external_task_id</span><span class="p">,</span>
             <span class="n">allowed_states</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
             <span class="n">execution_delta</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+            <span class="n">execution_date_fn</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
             <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
         <span class="nb">super</span><span class="p">(</span><span class="n">ExternalTaskSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">allowed_states</span> <span class="o">=</span> <span class="n">allowed_states</span> <span class="ow">or</span> <span class="p">[</span><span class="n">State</span><span class="o">.</span><span class="n">SUCCESS</span><span class="p">]</span>
+        <span class="k">if</span> <span class="n">execution_delta</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span> <span class="ow">and</span> <span class="n">execution_date_fn</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                <span class="s1">&#39;Only one of `execution_date` or `execution_date_fn` may&#39;</span>
+                <span class="s1">&#39;be provided to ExternalTaskSensor; not both.&#39;</span><span class="p">)</span>
+
         <span class="bp">self</span><span class="o">.</span><span class="n">execution_delta</span> <span class="o">=</span> <span class="n">execution_delta</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">execution_date_fn</span> <span class="o">=</span> <span class="n">execution_date_fn</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">external_dag_id</span> <span class="o">=</span> <span class="n">external_dag_id</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">external_task_id</span> <span class="o">=</span> <span class="n">external_task_id</span>
 
     <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
         <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_delta</span><span class="p">:</span>
             <span class="n">dttm</span> <span class="o">=</span> <span class="n">context</span><span class="p">[</span><span class="s1">&#39;execution_date&#39;</span><span class="p">]</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_delta</span>
+        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date_fn</span><span class="p">:</span>
+            <span class="n">dttm</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date_fn</span><span class="p">(</span><span class="n">context</span><span class="p">[</span><span class="s1">&#39;execution_date&#39;</span><span class="p">])</span>
         <span class="k">else</span><span class="p">:</span>
             <span class="n">dttm</span> <span class="o">=</span> <span class="n">context</span><span class="p">[</span><span class="s1">&#39;execution_date&#39;</span><span class="p">]</span>
 
@@ -346,15 +376,91 @@
         <span class="k">return</span> <span class="n">count</span>
 
 
+<span class="k">class</span> <span class="nc">NamedHivePartitionSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Waits for a set of partitions to show up in Hive.</span>
+
+<span class="sd">    :param partition_names: List of fully qualified names of the</span>
+<span class="sd">        partitions to wait for. A fully qualified name is of the</span>
+<span class="sd">        form schema.table/pk1=pv1/pk2=pv2, for example,</span>
+<span class="sd">        default.users/ds=2016-01-01. This is passed as is to the metastore</span>
+<span class="sd">        Thrift client &quot;get_partitions_by_name&quot; method. Note that</span>
+<span class="sd">        you cannot use logical operators as in HivePartitionSensor.</span>
+<span class="sd">    :type partition_names: list of strings</span>
+<span class="sd">    :param metastore_conn_id: reference to the metastore thrift service</span>
+<span class="sd">        connection id</span>
+<span class="sd">    :type metastore_conn_id: str</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;partition_names&#39;</span><span class="p">,</span> <span class="p">)</span>
+
+    <span class="nd">@apply_defaults</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+            <span class="bp">self</span><span class="p">,</span>
+            <span class="n">partition_names</span><span class="p">,</span>
+            <span class="n">metastore_conn_id</span><span class="o">=</span><span class="s1">&#39;metastore_default&#39;</span><span class="p">,</span>
+            <span class="n">poke_interval</span><span class="o">=</span><span class="mi">60</span><span class="o">*</span><span class="mi">3</span><span class="p">,</span>
+            <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+            <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">NamedHivePartitionSensor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span>
+            <span class="n">poke_interval</span><span class="o">=</span><span class="n">poke_interval</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">partition_names</span><span class="p">,</span> <span class="nb">basestring</span><span class="p">):</span>
+            <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s1">&#39;partition_names must be an array of strings&#39;</span><span class="p">)</span>
+
+        <span class="k">for</span> <span class="n">partition_name</span> <span class="ow">in</span> <span class="n">partition_names</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">parse_partition_name</span><span class="p">(</span><span class="n">partition_name</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">metastore_conn_id</span> <span class="o">=</span> <span class="n">metastore_conn_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">partition_names</span> <span class="o">=</span> <span class="n">partition_names</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">next_poke_idx</span> <span class="o">=</span> <span class="mi">0</span>
+
+    <span class="k">def</span> <span class="nf">parse_partition_name</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">partition</span><span class="p">):</span>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">schema</span><span class="p">,</span> <span class="n">table_partition</span> <span class="o">=</span> <span class="n">partition</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">)</span>
+            <span class="n">table</span><span class="p">,</span> <span class="n">partition</span> <span class="o">=</span> <span class="n">table_partition</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
+            <span class="k">return</span> <span class="n">schema</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">partition</span>
+        <span class="k">except</span> <span class="ne">ValueError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;Could not parse &#39;</span> <span class="o">+</span> <span class="n">partition</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+
+        <span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s1">&#39;hook&#39;</span><span class="p">):</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">airflow</span><span class="o">.</span><span class="n">hooks</span><span class="o">.</span><span class="n">hive_hooks</span><span class="o">.</span><span class="n">HiveMetastoreHook</span><span class="p">(</span>
+                <span class="n">metastore_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">metastore_conn_id</span><span class="p">)</span>
+
+        <span class="k">def</span> <span class="nf">poke_partition</span><span class="p">(</span><span class="n">partition</span><span class="p">):</span>
+
+            <span class="n">schema</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">partition</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parse_partition_name</span><span class="p">(</span><span class="n">partition</span><span class="p">)</span>
+
+            <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+                <span class="s1">&#39;Poking for {schema}.{table}/{partition}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">())</span>
+            <span class="p">)</span>
+            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">check_for_named_partition</span><span class="p">(</span>
+                <span class="n">schema</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">partition</span><span class="p">)</span>
+
+        <span class="k">while</span> <span class="bp">self</span><span class="o">.</span><span class="n">next_poke_idx</span> <span class="o">&lt;</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">partition_names</span><span class="p">):</span>
+            <span class="k">if</span> <span class="n">poke_partition</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">partition_names</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">next_poke_idx</span><span class="p">]):</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">next_poke_idx</span> <span class="o">+=</span> <span class="mi">1</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="k">return</span> <span class="bp">False</span>
+
+        <span class="k">return</span> <span class="bp">True</span>
+
+
 <span class="k">class</span> <span class="nc">HivePartitionSensor</span><span class="p">(</span><span class="n">BaseSensorOperator</span><span class="p">):</span>
     <span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">    Waits for a partition to show up in Hive</span>
+<span class="sd">    Waits for a partition to show up in Hive.</span>
+
+<span class="sd">    Note: Because @partition supports general logical operators, it</span>
+<span class="sd">    can be inefficient. Consider using NamedHivePartitionSensor instead if</span>
+<span class="sd">    you don&#39;t need the full flexibility of HivePartitionSensor.</span>
 
 <span class="sd">    :param table: The name of the table to wait for, supports the dot</span>
 <span class="sd">        notation (my_database.my_table)</span>
 <span class="sd">    :type table: string</span>
 <span class="sd">    :param partition: The partition clause to wait for. This is passed as</span>
-<span class="sd">        is to the Metastore Thrift client &quot;get_partitions_by_filter&quot; method,</span>
+<span class="sd">        is to the metastore Thrift client &quot;get_partitions_by_filter&quot; method,</span>
 <span class="sd">        and apparently supports SQL like notation as in `ds=&#39;2015-01-01&#39;</span>
 <span class="sd">        AND type=&#39;value&#39;` and &gt; &lt; sings as in &quot;ds&gt;=2015-01-01&quot;</span>
 <span class="sd">    :type partition: string</span>
@@ -388,7 +494,7 @@
             <span class="s1">&#39;Poking for table {self.schema}.{self.table}, &#39;</span>
             <span class="s1">&#39;partition {self.partition}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
         <span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s1">&#39;hook&#39;</span><span class="p">):</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">HiveMetastoreHook</span><span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">airflow</span><span class="o">.</span><span class="n">hooks</span><span class="o">.</span><span class="n">hive_hooks</span><span class="o">.</span><span class="n">HiveMetastoreHook</span><span class="p">(</span>
                 <span class="n">metastore_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">metastore_conn_id</span><span class="p">)</span>
         <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">check_for_partition</span><span class="p">(</span>
             <span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">table</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">partition</span><span class="p">)</span>
@@ -411,7 +517,8 @@
         <span class="bp">self</span><span class="o">.</span><span class="n">hdfs_conn_id</span> <span class="o">=</span> <span class="n">hdfs_conn_id</span>
 
     <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
-        <span class="n">sb</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">HDFSHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">hdfs_conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+        <span class="kn">import</span> <span class="nn">airflow.hooks.hdfs_hook</span>
+        <span class="n">sb</span> <span class="o">=</span> <span class="n">airflow</span><span class="o">.</span><span class="n">hooks</span><span class="o">.</span><span class="n">hdfs_hook</span><span class="o">.</span><span class="n">HDFSHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">hdfs_conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">&quot;snakebite&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">WARNING</span><span class="p">)</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
             <span class="s1">&#39;Poking for file {self.filepath} &#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
@@ -439,7 +546,7 @@
         <span class="bp">self</span><span class="o">.</span><span class="n">webhdfs_conn_id</span> <span class="o">=</span> <span class="n">webhdfs_conn_id</span>
 
     <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
-        <span class="n">c</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">WebHDFSHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">webhdfs_conn_id</span><span class="p">)</span>
+        <span class="n">c</span> <span class="o">=</span> <span class="n">airflow</span><span class="o">.</span><span class="n">hooks</span><span class="o">.</span><span class="n">webhdfs_hook</span><span class="o">.</span><span class="n">WebHDFSHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">webhdfs_conn_id</span><span class="p">)</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
             <span class="s1">&#39;Poking for file {self.filepath} &#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
         <span class="k">return</span> <span class="n">c</span><span class="o">.</span><span class="n">check_for_path</span><span class="p">(</span><span class="n">hdfs_path</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">filepath</span><span class="p">)</span>
@@ -495,7 +602,8 @@
         <span class="n">session</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
 
     <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
-        <span class="n">hook</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">S3Hook</span><span class="p">(</span><span class="n">s3_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">s3_conn_id</span><span class="p">)</span>
+        <span class="kn">import</span> <span class="nn">airflow.hooks.S3_hook</span>
+        <span class="n">hook</span> <span class="o">=</span> <span class="n">airflow</span><span class="o">.</span><span class="n">hooks</span><span class="o">.</span><span class="n">S3_hook</span><span class="o">.</span><span class="n">S3Hook</span><span class="p">(</span><span class="n">s3_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">s3_conn_id</span><span class="p">)</span>
         <span class="n">full_url</span> <span class="o">=</span> <span class="s2">&quot;s3://&quot;</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">bucket_name</span> <span class="o">+</span> <span class="s2">&quot;/&quot;</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">bucket_key</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Poking for key : {full_url}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
         <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">wildcard_match</span><span class="p">:</span>
@@ -547,7 +655,8 @@
     <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Poking for prefix : {self.prefix}</span><span class="se">\n</span><span class="s1">&#39;</span>
                      <span class="s1">&#39;in bucket s3://{self.bucket_name}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
-        <span class="n">hook</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">S3Hook</span><span class="p">(</span><span class="n">s3_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">s3_conn_id</span><span class="p">)</span>
+        <span class="kn">import</span> <span class="nn">airflow.hooks.S3_hook</span>
+        <span class="n">hook</span> <span class="o">=</span> <span class="n">airflow</span><span class="o">.</span><span class="n">hooks</span><span class="o">.</span><span class="n">S3_hook</span><span class="o">.</span><span class="n">S3Hook</span><span class="p">(</span><span class="n">s3_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">s3_conn_id</span><span class="p">)</span>
         <span class="k">return</span> <span class="n">hook</span><span class="o">.</span><span class="n">check_for_prefix</span><span class="p">(</span>
             <span class="n">prefix</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">prefix</span><span class="p">,</span>
             <span class="n">delimiter</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delimiter</span><span class="p">,</span>
@@ -639,7 +748,7 @@
         <span class="bp">self</span><span class="o">.</span><span class="n">extra_options</span> <span class="o">=</span> <span class="n">extra_options</span> <span class="ow">or</span> <span class="p">{}</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">response_check</span> <span class="o">=</span> <span class="n">response_check</span>
 
-        <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">HttpHook</span><span class="p">(</span><span class="n">method</span><span class="o">=</span><span class="s1">&#39;GET&#39;</span><span class="p">,</span> <span class="n">http_conn_id</span><span class="o">=</span><span class="n">http_conn_id</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">hooks</span><span class="o">.</span><span class="n">http_hook</span><span class="o">.</span><span class="n">HttpHook</span><span class="p">(</span><span class="n">method</span><span class="o">=</span><span class="s1">&#39;GET&#39;</span><span class="p">,</span> <span class="n">http_conn_id</span><span class="o">=</span><span class="n">http_conn_id</span><span class="p">)</span>
 
     <span class="k">def</span> <span class="nf">poke</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Poking: &#39;</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">endpoint</span><span class="p">)</span>
@@ -652,7 +761,7 @@
                 <span class="c1"># run content check on response</span>
                 <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">response_check</span><span class="p">(</span><span class="n">response</span><span class="p">)</span>
         <span class="k">except</span> <span class="n">AirflowException</span> <span class="k">as</span> <span class="n">ae</span><span class="p">:</span>
-            <span class="k">if</span> <span class="n">ae</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">&quot;404&quot;</span><span class="p">):</span>
+            <span class="k">if</span> <span class="nb">str</span><span class="p">(</span><span class="n">ae</span><span class="p">)</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">&quot;404&quot;</span><span class="p">):</span>
                 <span class="k">return</span> <span class="bp">False</span>
 
             <span class="k">raise</span> <span class="n">ae</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/bash_operator.html
----------------------------------------------------------------------
diff --git a/_modules/bash_operator.html b/_modules/bash_operator.html
index bda570d..d38a3f8 100644
--- a/_modules/bash_operator.html
+++ b/_modules/bash_operator.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for bash_operator</h1><div class="highlight"><pre>
-<span></span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+
 <span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">bytes</span>
 <span class="kn">import</span> <span class="nn">logging</span>
 <span class="kn">from</span> <span class="nn">subprocess</span> <span class="kn">import</span> <span class="n">Popen</span><span class="p">,</span> <span class="n">STDOUT</span><span class="p">,</span> <span class="n">PIPE</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/dagrun_operator.html
----------------------------------------------------------------------
diff --git a/_modules/dagrun_operator.html b/_modules/dagrun_operator.html
index be98472..c6725bb 100644
--- a/_modules/dagrun_operator.html
+++ b/_modules/dagrun_operator.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for dagrun_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">datetime</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">datetime</span>
 <span class="kn">import</span> <span class="nn">logging</span>
 
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span><span class="p">,</span> <span class="n">DagRun</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/dbapi_hook.html
----------------------------------------------------------------------
diff --git a/_modules/dbapi_hook.html b/_modules/dbapi_hook.html
index 4a6a3a0..e608a23 100644
--- a/_modules/dbapi_hook.html
+++ b/_modules/dbapi_hook.html
@@ -137,12 +137,26 @@
            <div itemprop="articleBody">
             
   <h1>Source code for dbapi_hook</h1><div class="highlight"><pre>
-<span></span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
 <span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
 <span class="kn">from</span> <span class="nn">past.builtins</span> <span class="kn">import</span> <span class="nb">basestring</span>
 <span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">datetime</span>
 <span class="kn">import</span> <span class="nn">numpy</span>
 <span class="kn">import</span> <span class="nn">logging</span>
+<span class="kn">import</span> <span class="nn">sys</span>
 
 <span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="kn">import</span> <span class="n">BaseHook</span>
 <span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span>
@@ -192,6 +206,8 @@
 <span class="sd">        :param parameters: The parameters to render the SQL query with.</span>
 <span class="sd">        :type parameters: mapping or iterable</span>
 <span class="sd">        &#39;&#39;&#39;</span>
+        <span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">version_info</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">&lt;</span> <span class="mi">3</span><span class="p">:</span>
+            <span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span>
         <span class="kn">import</span> <span class="nn">pandas.io.sql</span> <span class="kn">as</span> <span class="nn">psql</span>
         <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
         <span class="n">df</span> <span class="o">=</span> <span class="n">psql</span><span class="o">.</span><span class="n">read_sql</span><span class="p">(</span><span class="n">sql</span><span class="p">,</span> <span class="n">con</span><span class="o">=</span><span class="n">conn</span><span class="p">,</span> <span class="n">params</span><span class="o">=</span><span class="n">parameters</span><span class="p">)</span>
@@ -208,6 +224,8 @@
 <span class="sd">        :param parameters: The parameters to render the SQL query with.</span>
 <span class="sd">        :type parameters: mapping or iterable</span>
 <span class="sd">        &#39;&#39;&#39;</span>
+        <span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">version_info</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">&lt;</span> <span class="mi">3</span><span class="p">:</span>
+            <span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span>
         <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
         <span class="n">cur</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_cursor</span><span class="p">()</span>
         <span class="k">if</span> <span class="n">parameters</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span>
@@ -229,6 +247,8 @@
 <span class="sd">        :param parameters: The parameters to render the SQL query with.</span>
 <span class="sd">        :type parameters: mapping or iterable</span>
 <span class="sd">        &#39;&#39;&#39;</span>
+        <span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">version_info</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">&lt;</span> <span class="mi">3</span><span class="p">:</span>
+            <span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span>
         <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
         <span class="n">cur</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span>
         <span class="k">if</span> <span class="n">parameters</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span>
@@ -264,6 +284,8 @@
 
         <span class="n">cur</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span>
         <span class="k">for</span> <span class="n">s</span> <span class="ow">in</span> <span class="n">sql</span><span class="p">:</span>
+            <span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">version_info</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">&lt;</span> <span class="mi">3</span><span class="p">:</span>
+                <span class="n">s</span> <span class="o">=</span> <span class="n">s</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span>
             <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">s</span><span class="p">)</span>
             <span class="k">if</span> <span class="n">parameters</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span>
                 <span class="n">cur</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">s</span><span class="p">,</span> <span class="n">parameters</span><span class="p">)</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/druid_hook.html
----------------------------------------------------------------------
diff --git a/_modules/druid_hook.html b/_modules/druid_hook.html
index 1508432..f5b2a4d 100644
--- a/_modules/druid_hook.html
+++ b/_modules/druid_hook.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for druid_hook</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">print_function</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">print_function</span>
 <span class="kn">import</span> <span class="nn">logging</span>
 <span class="kn">import</span> <span class="nn">json</span>
 <span class="kn">import</span> <span class="nn">time</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/dummy_operator.html
----------------------------------------------------------------------
diff --git a/_modules/dummy_operator.html b/_modules/dummy_operator.html
index 4d8943a..a531fd1 100644
--- a/_modules/dummy_operator.html
+++ b/_modules/dummy_operator.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for dummy_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/email_operator.html
----------------------------------------------------------------------
diff --git a/_modules/email_operator.html b/_modules/email_operator.html
index 59f4127..6bb4434 100644
--- a/_modules/email_operator.html
+++ b/_modules/email_operator.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for email_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.email</span> <span class="kn">import</span> <span class="n">send_email</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/ftp_hook.html
----------------------------------------------------------------------
diff --git a/_modules/ftp_hook.html b/_modules/ftp_hook.html
index 1cc038a..19a7498 100644
--- a/_modules/ftp_hook.html
+++ b/_modules/ftp_hook.html
@@ -161,8 +161,8 @@
 
 
 <span class="k">def</span> <span class="nf">mlsd</span><span class="p">(</span><span class="n">conn</span><span class="p">,</span> <span class="n">path</span><span class="o">=</span><span class="s2">&quot;&quot;</span><span class="p">,</span> <span class="n">facts</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
-    <span class="sd">&#39;&#39;&#39;</span>
-<span class="sd">    BACKPORT FROM PYTHON3 FTPLIB</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    BACKPORT FROM PYTHON3 FTPLIB.</span>
 
 <span class="sd">    List a directory in a standardized format by using MLSD</span>
 <span class="sd">    command (RFC-3659). If path is omitted the current directory</span>
@@ -174,7 +174,7 @@
 <span class="sd">    First element is the file name, the second one is a dictionary</span>
 <span class="sd">    including a variable number of &quot;facts&quot; depending on the server</span>
 <span class="sd">    and whether &quot;facts&quot; argument has been provided.</span>
-<span class="sd">    &#39;&#39;&#39;</span>
+<span class="sd">    &quot;&quot;&quot;</span>
     <span class="n">facts</span> <span class="o">=</span> <span class="n">facts</span> <span class="ow">or</span> <span class="p">[]</span>
     <span class="k">if</span> <span class="n">facts</span><span class="p">:</span>
         <span class="n">conn</span><span class="o">.</span><span class="n">sendcmd</span><span class="p">(</span><span class="s2">&quot;OPTS MLST &quot;</span> <span class="o">+</span> <span class="s2">&quot;;&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">facts</span><span class="p">)</span> <span class="o">+</span> <span class="s2">&quot;;&quot;</span><span class="p">)</span>
@@ -194,7 +194,6 @@
 
 
 <div class="viewcode-block" id="FTPHook"><a class="viewcode-back" href="../code.html#airflow.contrib.hooks.FTPHook">[docs]</a><span class="k">class</span> <span class="nc">FTPHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span>
-
     <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Interact with FTP.</span>
 
@@ -206,6 +205,13 @@
         <span class="bp">self</span><span class="o">.</span><span class="n">ftp_conn_id</span> <span class="o">=</span> <span class="n">ftp_conn_id</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="bp">None</span>
 
+    <span class="k">def</span> <span class="nf">__enter__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">return</span> <span class="bp">self</span>
+
+    <span class="k">def</span> <span class="nf">__exit__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">exc_type</span><span class="p">,</span> <span class="n">exc_val</span><span class="p">,</span> <span class="n">exc_tb</span><span class="p">):</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">close_conn</span><span class="p">()</span>
+
 <div class="viewcode-block" id="FTPHook.get_conn"><a class="viewcode-back" href="../code.html#airflow.contrib.hooks.FTPHook.get_conn">[docs]</a>    <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Returns a FTP connection object</span>
@@ -219,7 +225,7 @@
 <div class="viewcode-block" id="FTPHook.close_conn"><a class="viewcode-back" href="../code.html#airflow.contrib.hooks.FTPHook.close_conn">[docs]</a>    <span class="k">def</span> <span class="nf">close_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Closes the connection. An error will occur if the</span>
-<span class="sd">        connection wasnt ever opened.</span>
+<span class="sd">        connection wasn&#39;t ever opened.</span>
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span>
         <span class="n">conn</span><span class="o">.</span><span class="n">quit</span><span class="p">()</span></div>
@@ -301,7 +307,7 @@
         <span class="n">conn</span><span class="o">.</span><span class="n">cwd</span><span class="p">(</span><span class="n">remote_path</span><span class="p">)</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Retrieving file from FTP: {}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">remote_full_path</span><span class="p">))</span>
         <span class="n">conn</span><span class="o">.</span><span class="n">retrbinary</span><span class="p">(</span><span class="s1">&#39;RETR </span><span class="si">%s</span><span class="s1">&#39;</span> <span class="o">%</span> <span class="n">remote_file_name</span><span class="p">,</span> <span class="n">output_handle</span><span class="o">.</span><span class="n">write</span><span class="p">)</span>
-        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Finished etrieving file from FTP: {}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Finished retrieving file from FTP: {}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
             <span class="n">remote_full_path</span><span class="p">))</span>
 
         <span class="k">if</span> <span class="n">is_path</span><span class="p">:</span>
@@ -338,7 +344,7 @@
 
 <div class="viewcode-block" id="FTPHook.delete_file"><a class="viewcode-back" href="../code.html#airflow.contrib.hooks.FTPHook.delete_file">[docs]</a>    <span class="k">def</span> <span class="nf">delete_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">        Removes a file on the FTP Server</span>
+<span class="sd">        Removes a file on the FTP Server.</span>
 
 <span class="sd">        :param path: full path to the remote file</span>
 <span class="sd">        :type path: str</span>
@@ -356,7 +362,7 @@
 
     <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">        Returns a FTPS connection object</span>
+<span class="sd">        Returns a FTPS connection object.</span>
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
             <span class="n">params</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">ftp_conn_id</span><span class="p">)</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/generic_transfer.html
----------------------------------------------------------------------
diff --git a/_modules/generic_transfer.html b/_modules/generic_transfer.html
index 4e2cf14..ed7f740 100644
--- a/_modules/generic_transfer.html
+++ b/_modules/generic_transfer.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for generic_transfer</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
 
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/hive_hooks.html
----------------------------------------------------------------------
diff --git a/_modules/hive_hooks.html b/_modules/hive_hooks.html
index 7f6cc2d..e25ec97 100644
--- a/_modules/hive_hooks.html
+++ b/_modules/hive_hooks.html
@@ -155,11 +155,13 @@
 <span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">print_function</span>
 <span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">zip</span>
 <span class="kn">from</span> <span class="nn">past.builtins</span> <span class="kn">import</span> <span class="nb">basestring</span>
+
 <span class="kn">import</span> <span class="nn">unicodecsv</span> <span class="kn">as</span> <span class="nn">csv</span>
 <span class="kn">import</span> <span class="nn">logging</span>
 <span class="kn">import</span> <span class="nn">re</span>
 <span class="kn">import</span> <span class="nn">subprocess</span>
 <span class="kn">from</span> <span class="nn">tempfile</span> <span class="kn">import</span> <span class="n">NamedTemporaryFile</span>
+<span class="kn">import</span> <span class="nn">hive_metastore</span>
 
 <span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span>
 <span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="kn">import</span> <span class="n">BaseHook</span>
@@ -180,6 +182,7 @@
 <span class="sd">    Note that you can also set default hive CLI parameters using the</span>
 <span class="sd">    ``hive_cli_params`` to be used in your connection as in</span>
 <span class="sd">    ``{&quot;hive_cli_params&quot;: &quot;-hiveconf mapred.job.tracker=some.jobtracker:444&quot;}``</span>
+<span class="sd">    Parameters passed here can be overridden by run_cli&#39;s hive_conf param</span>
 
 <span class="sd">    The extra connection parameter ``auth`` gets passed as in the ``jdbc``</span>
 <span class="sd">    connection string as is.</span>
@@ -196,9 +199,17 @@
         <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="n">conn</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">run_as</span> <span class="o">=</span> <span class="n">run_as</span>
 
-<div class="viewcode-block" id="HiveCliHook.run_cli"><a class="viewcode-back" href="../code.html#airflow.hooks.HiveCliHook.run_cli">[docs]</a>    <span class="k">def</span> <span class="nf">run_cli</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">hql</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="bp">True</span><span class="p">):</span>
+<div class="viewcode-block" id="HiveCliHook.run_cli"><a class="viewcode-back" href="../code.html#airflow.hooks.HiveCliHook.run_cli">[docs]</a>    <span class="k">def</span> <span class="nf">run_cli</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">hql</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span> <span class="n">hive_conf</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">        Run an hql statement using the hive cli</span>
+<span class="sd">        Run an hql statement using the hive cli. If hive_conf is specified it should be a</span>
+<span class="sd">        dict and the entries will be set as key/value pairs in HiveConf</span>
+
+
+<span class="sd">        :param hive_conf: if specified these key value pairs will be passed to hive as</span>
+<span class="sd">            ``-hiveconf &quot;key&quot;=&quot;value&quot;``. Note that they will be passed after the</span>
+<span class="sd">            ``hive_cli_params`` and thus will override whatever values are specified in</span>
+<span class="sd">            the database.</span>
+<span class="sd">        :type hive_conf: dict</span>
 
 <span class="sd">        &gt;&gt;&gt; hh = HiveCliHook()</span>
 <span class="sd">        &gt;&gt;&gt; result = hh.run_cli(&quot;USE airflow;&quot;)</span>
@@ -246,6 +257,10 @@
                     <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">:</span>
                         <span class="n">cmd_extra</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">&#39;-p&#39;</span><span class="p">,</span> <span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">]</span>
 
+                <span class="n">hive_conf</span> <span class="o">=</span> <span class="n">hive_conf</span> <span class="ow">or</span> <span class="p">{}</span>
+                <span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">hive_conf</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
+                    <span class="n">cmd_extra</span> <span class="o">+=</span> <span class="p">[</span><span class="s1">&#39;-hiveconf&#39;</span><span class="p">,</span> <span class="s1">&#39;{0}={1}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">value</span><span class="p">)]</span>
+
                 <span class="n">hive_cmd</span> <span class="o">=</span> <span class="p">[</span><span class="n">hive_bin</span><span class="p">,</span> <span class="s1">&#39;-f&#39;</span><span class="p">,</span> <span class="n">fname</span><span class="p">]</span> <span class="o">+</span> <span class="n">cmd_extra</span>
 
                 <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">hive_cli_params</span><span class="p">:</span>
@@ -446,7 +461,17 @@
         <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">metastore</span>
 
 <div class="viewcode-block" id="HiveMetastoreHook.check_for_partition"><a class="viewcode-back" href="../code.html#airflow.hooks.HiveMetastoreHook.check_for_partition">[docs]</a>    <span class="k">def</span> <span class="nf">check_for_partition</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">schema</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">partition</span><span class="p">):</span>
-        <span class="sd">&quot;&quot;&quot;Checks whether a partition exists</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Checks whether a partition exists</span>
+
+<span class="sd">        :param schema: Name of hive schema (database) @table belongs to</span>
+<span class="sd">        :type schema: string</span>
+<span class="sd">        :param table: Name of hive table @partition belongs to</span>
+<span class="sd">        :type schema: string</span>
+<span class="sd">        :partition: Expression that matches the partitions to check for</span>
+<span class="sd">            (eg `a = &#39;b&#39; AND c = &#39;d&#39;`)</span>
+<span class="sd">        :type schema: string</span>
+<span class="sd">        :rtype: boolean</span>
 
 <span class="sd">        &gt;&gt;&gt; hh = HiveMetastoreHook()</span>
 <span class="sd">        &gt;&gt;&gt; t = &#39;static_babynames_partitioned&#39;</span>
@@ -462,6 +487,35 @@
         <span class="k">else</span><span class="p">:</span>
             <span class="k">return</span> <span class="bp">False</span></div>
 
+<div class="viewcode-block" id="HiveMetastoreHook.check_for_named_partition"><a class="viewcode-back" href="../code.html#airflow.hooks.HiveMetastoreHook.check_for_named_partition">[docs]</a>    <span class="k">def</span> <span class="nf">check_for_named_partition</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">schema</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">partition_name</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Checks whether a partition with a given name exists</span>
+
+<span class="sd">        :param schema: Name of hive schema (database) @table belongs to</span>
+<span class="sd">        :type schema: string</span>
+<span class="sd">        :param table: Name of hive table @partition belongs to</span>
+<span class="sd">        :type schema: string</span>
+<span class="sd">        :partition: Name of the partitions to check for (eg `a=b/c=d`)</span>
+<span class="sd">        :type schema: string</span>
+<span class="sd">        :rtype: boolean</span>
+
+<span class="sd">        &gt;&gt;&gt; hh = HiveMetastoreHook()</span>
+<span class="sd">        &gt;&gt;&gt; t = &#39;static_babynames_partitioned&#39;</span>
+<span class="sd">        &gt;&gt;&gt; hh.check_for_named_partition(&#39;airflow&#39;, t, &quot;ds=2015-01-01&quot;)</span>
+<span class="sd">        True</span>
+<span class="sd">        &gt;&gt;&gt; hh.check_for_named_partition(&#39;airflow&#39;, t, &quot;ds=xxx&quot;)</span>
+<span class="sd">        False</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">metastore</span><span class="o">.</span><span class="n">_oprot</span><span class="o">.</span><span class="n">trans</span><span class="o">.</span><span class="n">open</span><span class="p">()</span>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">metastore</span><span class="o">.</span><span class="n">get_partition_by_name</span><span class="p">(</span>
+                <span class="n">schema</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">partition_name</span><span class="p">)</span>
+            <span class="k">return</span> <span class="bp">True</span>
+        <span class="k">except</span> <span class="n">hive_metastore</span><span class="o">.</span><span class="n">ttypes</span><span class="o">.</span><span class="n">NoSuchObjectException</span><span class="p">:</span>
+            <span class="k">return</span> <span class="bp">False</span>
+        <span class="k">finally</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">metastore</span><span class="o">.</span><span class="n">_oprot</span><span class="o">.</span><span class="n">trans</span><span class="o">.</span><span class="n">close</span><span class="p">()</span></div>
+
 <div class="viewcode-block" id="HiveMetastoreHook.get_table"><a class="viewcode-back" href="../code.html#airflow.hooks.HiveMetastoreHook.get_table">[docs]</a>    <span class="k">def</span> <span class="nf">get_table</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table_name</span><span class="p">,</span> <span class="n">db</span><span class="o">=</span><span class="s1">&#39;default&#39;</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;Get a metastore table object</span>
 
@@ -604,6 +658,7 @@
             <span class="n">database</span><span class="o">=</span><span class="n">db</span><span class="o">.</span><span class="n">schema</span> <span class="ow">or</span> <span class="s1">&#39;default&#39;</span><span class="p">)</span>
 
     <span class="k">def</span> <span class="nf">get_results</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">hql</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="s1">&#39;default&#39;</span><span class="p">,</span> <span class="n">arraysize</span><span class="o">=</span><span class="mi">1000</span><span class="p">):</span>
+        <span class="kn">from</span> <span class="nn">impala.error</span> <span class="kn">import</span> <span class="n">ProgrammingError</span>
         <span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span> <span class="k">as</span> <span class="n">conn</span><span class="p">:</span>
             <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">hql</span><span class="p">,</span> <span class="nb">basestring</span><span class="p">):</span>
                 <span class="n">hql</span> <span class="o">=</span> <span class="p">[</span><span class="n">hql</span><span class="p">]</span>
@@ -611,15 +666,22 @@
                 <span class="s1">&#39;data&#39;</span><span class="p">:</span> <span class="p">[],</span>
                 <span class="s1">&#39;header&#39;</span><span class="p">:</span> <span class="p">[],</span>
             <span class="p">}</span>
+            <span class="n">cur</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span>
             <span class="k">for</span> <span class="n">statement</span> <span class="ow">in</span> <span class="n">hql</span><span class="p">:</span>
-                <span class="k">with</span> <span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span> <span class="k">as</span> <span class="n">cur</span><span class="p">:</span>
-                    <span class="n">cur</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">statement</span><span class="p">)</span>
+                <span class="n">cur</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">statement</span><span class="p">)</span>
+                <span class="n">records</span> <span class="o">=</span> <span class="p">[]</span>
+                <span class="k">try</span><span class="p">:</span>
+                    <span class="c1"># impala Lib raises when no results are returned</span>
+                    <span class="c1"># we&#39;re silencing here as some statements in the list</span>
+                    <span class="c1"># may be `SET` or DDL</span>
                     <span class="n">records</span> <span class="o">=</span> <span class="n">cur</span><span class="o">.</span><span class="n">fetchall</span><span class="p">()</span>
-                    <span class="k">if</span> <span class="n">records</span><span class="p">:</span>
-                        <span class="n">results</span> <span class="o">=</span> <span class="p">{</span>
-                            <span class="s1">&#39;data&#39;</span><span class="p">:</span> <span class="n">records</span><span class="p">,</span>
-                            <span class="s1">&#39;header&#39;</span><span class="p">:</span> <span class="n">cur</span><span class="o">.</span><span class="n">description</span><span class="p">,</span>
-                        <span class="p">}</span>
+                <span class="k">except</span> <span class="n">ProgrammingError</span><span class="p">:</span>
+                    <span class="n">logging</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;get_results returned no records&quot;</span><span class="p">)</span>
+                <span class="k">if</span> <span class="n">records</span><span class="p">:</span>
+                    <span class="n">results</span> <span class="o">=</span> <span class="p">{</span>
+                        <span class="s1">&#39;data&#39;</span><span class="p">:</span> <span class="n">records</span><span class="p">,</span>
+                        <span class="s1">&#39;header&#39;</span><span class="p">:</span> <span class="n">cur</span><span class="o">.</span><span class="n">description</span><span class="p">,</span>
+                    <span class="p">}</span>
             <span class="k">return</span> <span class="n">results</span>
 
     <span class="k">def</span> <span class="nf">to_csv</span><span class="p">(</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/hive_operator.html
----------------------------------------------------------------------
diff --git a/_modules/hive_operator.html b/_modules/hive_operator.html
index d32e3ad..de5e44b 100644
--- a/_modules/hive_operator.html
+++ b/_modules/hive_operator.html
@@ -137,12 +137,27 @@
            <div itemprop="articleBody">
             
   <h1>Source code for hive_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
 <span class="kn">import</span> <span class="nn">re</span>
 
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.operator_helpers</span> <span class="kn">import</span> <span class="n">context_to_airflow_vars</span>
 
 
 <div class="viewcode-block" id="HiveOperator"><a class="viewcode-back" href="../code.html#airflow.operators.HiveOperator">[docs]</a><span class="k">class</span> <span class="nc">HiveOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
@@ -201,7 +216,8 @@
     <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Executing: &#39;</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">hql</span><span class="p">)</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_hook</span><span class="p">()</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">run_cli</span><span class="p">(</span><span class="n">hql</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">hql</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">run_cli</span><span class="p">(</span><span class="n">hql</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">hql</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span>
+                          <span class="n">hive_conf</span><span class="o">=</span><span class="n">context_to_airflow_vars</span><span class="p">(</span><span class="n">context</span><span class="p">))</span>
 
     <span class="k">def</span> <span class="nf">dry_run</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_hook</span><span class="p">()</span>

[5/5] incubator-airflow-site git commit: Updating the disclaimer

[ma...@apache.org]

Updating the disclaimer


Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/commit/457b698d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/tree/457b698d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/diff/457b698d

Branch: refs/heads/asf-site
Commit: 457b698d9f92af370061d322267d85886e6cbfe4
Parents: 502934c
Author: Maxime Beauchemin <ma...@apache.org>
Authored: Fri Jul 1 14:54:14 2016 -0700
Committer: Maxime Beauchemin <ma...@apache.org>
Committed: Fri Jul 1 14:54:14 2016 -0700

----------------------------------------------------------------------
 .../contrib/executors/mesos_executor.html       |  16 +-
 .../contrib/operators/bigquery_operator.html    |  16 +-
 .../contrib/operators/bigquery_to_gcs.html      |  16 +-
 .../operators/gcs_download_operator.html        |  37 +++-
 _modules/airflow/executors/celery_executor.html |  16 +-
 _modules/airflow/executors/local_executor.html  |  16 +-
 .../airflow/executors/sequential_executor.html  |  16 +-
 _modules/airflow/macros.html                    |  40 ++++-
 _modules/airflow/macros/hive.html               |  20 ++-
 _modules/airflow/models.html                    | 111 +++++++++---
 _modules/airflow/operators/docker_operator.html |  16 +-
 _modules/airflow/operators/sensors.html         | 137 ++++++++++++--
 _modules/bash_operator.html                     |  16 +-
 _modules/dagrun_operator.html                   |  16 +-
 _modules/dbapi_hook.html                        |  24 ++-
 _modules/druid_hook.html                        |  16 +-
 _modules/dummy_operator.html                    |  16 +-
 _modules/email_operator.html                    |  16 +-
 _modules/ftp_hook.html                          |  22 ++-
 _modules/generic_transfer.html                  |  16 +-
 _modules/hive_hooks.html                        |  82 +++++++--
 _modules/hive_operator.html                     |  22 ++-
 _modules/hive_to_druid.html                     |  48 +++--
 _modules/hive_to_mysql.html                     |  21 ++-
 _modules/hive_to_samba_operator.html            |  19 +-
 _modules/http_hook.html                         |   2 +-
 _modules/http_operator.html                     |  18 +-
 _modules/mssql_hook.html                        |  25 ++-
 _modules/mssql_operator.html                    |  20 ++-
 _modules/mssql_to_hive.html                     |  19 +-
 _modules/mysql_hook.html                        |  16 +-
 _modules/mysql_operator.html                    |  20 ++-
 _modules/mysql_to_hive.html                     |  19 +-
 _modules/postgres_hook.html                     |  16 +-
 _modules/postgres_operator.html                 |  20 ++-
 _modules/presto_check_operator.html             |  18 +-
 _modules/presto_hook.html                       |  25 ++-
 _modules/python_operator.html                   |  16 +-
 _modules/qubole_operator.html                   |  20 ++-
 _modules/s3_to_hive_operator.html               |  19 +-
 _modules/sensors.html                           | 137 ++++++++++++--
 _modules/slack_operator.html                    |  16 +-
 _modules/sqlite_hook.html                       |  16 +-
 _modules/ssh_execute_operator.html              |  16 +-
 _modules/vertica_operator.html                  |  20 ++-
 _modules/vertica_to_hive.html                   |  20 ++-
 _modules/webhdfs_hook.html                      |  16 +-
 _sources/code.txt                               |   9 +
 _sources/concepts.txt                           |   4 +-
 _sources/faq.txt                                |  15 +-
 _sources/index.txt                              |   7 +-
 _sources/tutorial.txt                           |   6 +-
 code.html                                       | 180 ++++++++++++++++---
 concepts.html                                   |   4 +-
 faq.html                                        |  13 +-
 genindex.html                                   |  23 ++-
 index.html                                      |   8 +-
 objects.inv                                     | Bin 2236 -> 2246 bytes
 searchindex.js                                  |   2 +-
 tutorial.html                                   |   6 +-
 60 files changed, 1363 insertions(+), 219 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/airflow/contrib/executors/mesos_executor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/executors/mesos_executor.html b/_modules/airflow/contrib/executors/mesos_executor.html
index 65557d9..e2d8f61 100644
--- a/_modules/airflow/contrib/executors/mesos_executor.html
+++ b/_modules/airflow/contrib/executors/mesos_executor.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for airflow.contrib.executors.mesos_executor</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">future</span> <span class="kn">import</span> <span class="n">standard_library</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">future</span> <span class="kn">import</span> <span class="n">standard_library</span>
 <span class="n">standard_library</span><span class="o">.</span><span class="n">install_aliases</span><span class="p">()</span>
 <span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
 <span class="kn">import</span> <span class="nn">logging</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/airflow/contrib/operators/bigquery_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/bigquery_operator.html b/_modules/airflow/contrib/operators/bigquery_operator.html
index 830fd57..6038f2c 100644
--- a/_modules/airflow/contrib/operators/bigquery_operator.html
+++ b/_modules/airflow/contrib/operators/bigquery_operator.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for airflow.contrib.operators.bigquery_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
 
 <span class="kn">from</span> <span class="nn">airflow.contrib.hooks.bigquery_hook</span> <span class="kn">import</span> <span class="n">BigQueryHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/airflow/contrib/operators/bigquery_to_gcs.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/bigquery_to_gcs.html b/_modules/airflow/contrib/operators/bigquery_to_gcs.html
index 465128c..a3b8d18 100644
--- a/_modules/airflow/contrib/operators/bigquery_to_gcs.html
+++ b/_modules/airflow/contrib/operators/bigquery_to_gcs.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for airflow.contrib.operators.bigquery_to_gcs</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
 
 <span class="kn">from</span> <span class="nn">airflow.contrib.hooks.bigquery_hook</span> <span class="kn">import</span> <span class="n">BigQueryHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/airflow/contrib/operators/gcs_download_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/gcs_download_operator.html b/_modules/airflow/contrib/operators/gcs_download_operator.html
index 4af6020..dbf56fb 100644
--- a/_modules/airflow/contrib/operators/gcs_download_operator.html
+++ b/_modules/airflow/contrib/operators/gcs_download_operator.html
@@ -137,7 +137,22 @@
            <div itemprop="articleBody">
             
   <h1>Source code for airflow.contrib.operators.gcs_download_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
+<span class="kn">import</span> <span class="nn">sys</span>
 
 <span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcs_hook</span> <span class="kn">import</span> <span class="n">GoogleCloudStorageHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
@@ -147,7 +162,7 @@
     <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Downloads a file from Google Cloud Storage.</span>
 <span class="sd">    &quot;&quot;&quot;</span>
-    <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;bucket&#39;</span><span class="p">,</span><span class="s1">&#39;object&#39;</span><span class="p">,</span><span class="s1">&#39;filename&#39;</span><span class="p">,)</span>
+    <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;bucket&#39;</span><span class="p">,</span><span class="s1">&#39;object&#39;</span><span class="p">,</span><span class="s1">&#39;filename&#39;</span><span class="p">,</span><span class="s1">&#39;store_to_xcom_key&#39;</span><span class="p">,)</span>
     <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;.sql&#39;</span><span class="p">,)</span>
     <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#f0eee4&#39;</span>
 
@@ -156,7 +171,8 @@
         <span class="bp">self</span><span class="p">,</span>
         <span class="n">bucket</span><span class="p">,</span>
         <span class="nb">object</span><span class="p">,</span>
-        <span class="n">filename</span><span class="p">,</span>
+        <span class="n">filename</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
+        <span class="n">store_to_xcom_key</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
         <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="s1">&#39;google_cloud_storage_default&#39;</span><span class="p">,</span>
         <span class="n">delegate_to</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
         <span class="o">*</span><span class="n">args</span><span class="p">,</span>
@@ -171,7 +187,13 @@
 <span class="sd">        :type object: string</span>
 <span class="sd">        :param filename: The file path on the local file system (where the</span>
 <span class="sd">            operator is being executed) that the file should be downloaded to.</span>
+<span class="sd">            If false, the downloaded data will not be stored on the local file</span>
+<span class="sd">            system.</span>
 <span class="sd">        :type filename: string</span>
+<span class="sd">        :param store_to_xcom_key: If this param is set, the operator will push</span>
+<span class="sd">            the contents of the downloaded file to XCom with the key set in this</span>
+<span class="sd">            parameter. If false, the downloaded data will not be pushed to XCom.</span>
+<span class="sd">        :type store_to_xcom_key: string</span>
 <span class="sd">        :param google_cloud_storage_conn_id: The connection ID to use when</span>
 <span class="sd">            connecting to Google cloud storage.</span>
 <span class="sd">        :type google_cloud_storage_conn_id: string</span>
@@ -183,6 +205,7 @@
         <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span> <span class="o">=</span> <span class="n">bucket</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">object</span> <span class="o">=</span> <span class="nb">object</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">filename</span> <span class="o">=</span> <span class="n">filename</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">store_to_xcom_key</span> <span class="o">=</span> <span class="n">store_to_xcom_key</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span> <span class="o">=</span> <span class="n">google_cloud_storage_conn_id</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
 
@@ -190,7 +213,13 @@
         <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Executing download: </span><span class="si">%s</span><span class="s1">, </span><span class="si">%s</span><span class="s1">, </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">object</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">filename</span><span class="p">)</span>
         <span class="n">hook</span> <span class="o">=</span> <span class="n">GoogleCloudStorageHook</span><span class="p">(</span><span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span><span class="p">,</span>
                                       <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
-        <span class="k">print</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">download</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">object</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">filename</span><span class="p">))</span></div>
+        <span class="n">file_bytes</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">download</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">object</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">filename</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">store_to_xcom_key</span><span class="p">:</span>
+            <span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">getsizeof</span><span class="p">(</span><span class="n">file_bytes</span><span class="p">)</span> <span class="o">&lt;</span> <span class="mi">48000</span><span class="p">:</span>
+                <span class="n">context</span><span class="p">[</span><span class="s1">&#39;ti&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">xcom_push</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">store_to_xcom_key</span><span class="p">,</span> <span class="n">value</span><span class="o">=</span><span class="n">file_bytes</span><span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s1">&#39;The size of the downloaded file is too large to push to XCom!&#39;</span><span class="p">)</span>
+        <span class="k">print</span><span class="p">(</span><span class="n">file_bytes</span><span class="p">)</span></div>
 </pre></div>
 
            </div>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/airflow/executors/celery_executor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/executors/celery_executor.html b/_modules/airflow/executors/celery_executor.html
index a231e28..9601c12 100644
--- a/_modules/airflow/executors/celery_executor.html
+++ b/_modules/airflow/executors/celery_executor.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for airflow.executors.celery_executor</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">object</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">object</span>
 <span class="kn">import</span> <span class="nn">logging</span>
 <span class="kn">import</span> <span class="nn">subprocess</span>
 <span class="kn">import</span> <span class="nn">time</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/airflow/executors/local_executor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/executors/local_executor.html b/_modules/airflow/executors/local_executor.html
index 79c456d..1841cb4 100644
--- a/_modules/airflow/executors/local_executor.html
+++ b/_modules/airflow/executors/local_executor.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for airflow.executors.local_executor</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">multiprocessing</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">multiprocessing</span>
 <span class="kn">import</span> <span class="nn">subprocess</span>
 <span class="kn">import</span> <span class="nn">time</span>
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/airflow/executors/sequential_executor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/executors/sequential_executor.html b/_modules/airflow/executors/sequential_executor.html
index a393b0f..ce84843 100644
--- a/_modules/airflow/executors/sequential_executor.html
+++ b/_modules/airflow/executors/sequential_executor.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for airflow.executors.sequential_executor</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
 <span class="kn">import</span> <span class="nn">subprocess</span>
 
 <span class="kn">from</span> <span class="nn">airflow.executors.base_executor</span> <span class="kn">import</span> <span class="n">BaseExecutor</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/airflow/macros.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/macros.html b/_modules/airflow/macros.html
index 6b4e17f..78efd22 100644
--- a/_modules/airflow/macros.html
+++ b/_modules/airflow/macros.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for airflow.macros</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">absolute_import</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">absolute_import</span>
 <span class="kn">from</span> <span class="nn">random</span> <span class="kn">import</span> <span class="n">random</span>
 <span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span>
 <span class="kn">import</span> <span class="nn">dateutil</span>
@@ -187,11 +201,29 @@
     <span class="k">return</span> <span class="n">datetime</span><span class="o">.</span><span class="n">strptime</span><span class="p">(</span><span class="n">ds</span><span class="p">,</span> <span class="n">input_format</span><span class="p">)</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="n">output_format</span><span class="p">)</span></div>
 
 
-<div class="viewcode-block" id="integrate_plugins"><a class="viewcode-back" href="../../code.html#airflow.macros.integrate_plugins">[docs]</a><span class="k">def</span> <span class="nf">integrate_plugins</span><span class="p">():</span>
+<span class="k">def</span> <span class="nf">_integrate_plugins</span><span class="p">():</span>
     <span class="sd">&quot;&quot;&quot;Integrate plugins to the context&quot;&quot;&quot;</span>
+    <span class="kn">import</span> <span class="nn">sys</span>
     <span class="kn">from</span> <span class="nn">airflow.plugins_manager</span> <span class="kn">import</span> <span class="n">macros</span> <span class="k">as</span> <span class="n">_macros</span>
-    <span class="k">for</span> <span class="n">_macro</span> <span class="ow">in</span> <span class="n">_macros</span><span class="p">:</span>
-        <span class="nb">globals</span><span class="p">()[</span><span class="n">_macro</span><span class="o">.</span><span class="n">__name__</span><span class="p">]</span> <span class="o">=</span> <span class="n">_macro</span></div>
+    <span class="k">for</span> <span class="n">_macro_module</span> <span class="ow">in</span> <span class="n">_macros</span><span class="p">:</span>
+        <span class="n">sys</span><span class="o">.</span><span class="n">modules</span><span class="p">[</span><span class="n">_macro_module</span><span class="o">.</span><span class="n">__name__</span><span class="p">]</span> <span class="o">=</span> <span class="n">_macro_module</span>
+        <span class="nb">globals</span><span class="p">()[</span><span class="n">_macro_module</span><span class="o">.</span><span class="n">_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">_macro_module</span>
+
+
+        <span class="c1">##########################################################</span>
+        <span class="c1"># TODO FIXME Remove in Airflow 2.0</span>
+
+        <span class="kn">import</span> <span class="nn">os</span> <span class="kn">as</span> <span class="nn">_os</span>
+        <span class="k">if</span> <span class="ow">not</span> <span class="n">_os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;AIRFLOW_USE_NEW_IMPORTS&#39;</span><span class="p">,</span> <span class="bp">False</span><span class="p">):</span>
+            <span class="kn">from</span> <span class="nn">zope.deprecation</span> <span class="kn">import</span> <span class="n">deprecated</span> <span class="k">as</span> <span class="n">_deprecated</span>
+            <span class="k">for</span> <span class="n">_macro</span> <span class="ow">in</span> <span class="n">_macro_module</span><span class="o">.</span><span class="n">_objects</span><span class="p">:</span>
+                <span class="nb">globals</span><span class="p">()[</span><span class="n">_macro</span><span class="o">.</span><span class="n">__name__</span><span class="p">]</span> <span class="o">=</span> <span class="n">_deprecated</span><span class="p">(</span>
+                    <span class="n">_macro</span><span class="p">,</span>
+                    <span class="s2">&quot;Importing plugin macro &#39;{i}&#39; directly from &quot;</span>
+                    <span class="s2">&quot;&#39;airflow.macros&#39; has been deprecated. Please &quot;</span>
+                    <span class="s2">&quot;import from &#39;airflow.macros.[plugin_module]&#39; &quot;</span>
+                    <span class="s2">&quot;instead. Support for direct imports will be dropped &quot;</span>
+                    <span class="s2">&quot;entirely in Airflow 2.0.&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">i</span><span class="o">=</span><span class="n">_macro</span><span class="p">))</span>
 </pre></div>
 
            </div>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/airflow/macros/hive.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/macros/hive.html b/_modules/airflow/macros/hive.html
index 43cd2a0..92000d3 100644
--- a/_modules/airflow/macros/hive.html
+++ b/_modules/airflow/macros/hive.html
@@ -139,7 +139,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for airflow.macros.hive</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">datetime</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">datetime</span>
 
 
 <div class="viewcode-block" id="max_partition"><a class="viewcode-back" href="../../../code.html#airflow.macros.hive.max_partition">[docs]</a><span class="k">def</span> <span class="nf">max_partition</span><span class="p">(</span>
@@ -166,7 +180,7 @@
 <span class="sd">    &gt;&gt;&gt; max_partition(&#39;airflow.static_babynames_partitioned&#39;)</span>
 <span class="sd">    &#39;2015-01-01&#39;</span>
 <span class="sd">    &#39;&#39;&#39;</span>
-    <span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HiveMetastoreHook</span>
+    <span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveMetastoreHook</span>
     <span class="k">if</span> <span class="s1">&#39;.&#39;</span> <span class="ow">in</span> <span class="n">table</span><span class="p">:</span>
         <span class="n">schema</span><span class="p">,</span> <span class="n">table</span> <span class="o">=</span> <span class="n">table</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">)</span>
     <span class="n">hh</span> <span class="o">=</span> <span class="n">HiveMetastoreHook</span><span class="p">(</span><span class="n">metastore_conn_id</span><span class="o">=</span><span class="n">metastore_conn_id</span><span class="p">)</span>
@@ -219,7 +233,7 @@
 <span class="sd">    &gt;&gt;&gt; closest_ds_partition(tbl, &#39;2015-01-02&#39;)</span>
 <span class="sd">    &#39;2015-01-01&#39;</span>
 <span class="sd">    &#39;&#39;&#39;</span>
-    <span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HiveMetastoreHook</span>
+    <span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveMetastoreHook</span>
     <span class="k">if</span> <span class="s1">&#39;.&#39;</span> <span class="ow">in</span> <span class="n">table</span><span class="p">:</span>
         <span class="n">schema</span><span class="p">,</span> <span class="n">table</span> <span class="o">=</span> <span class="n">table</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">)</span>
     <span class="n">hh</span> <span class="o">=</span> <span class="n">HiveMetastoreHook</span><span class="p">(</span><span class="n">metastore_conn_id</span><span class="o">=</span><span class="n">metastore_conn_id</span><span class="p">)</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/airflow/models.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/models.html b/_modules/airflow/models.html
index aba90cd..d4ed8a6 100644
--- a/_modules/airflow/models.html
+++ b/_modules/airflow/models.html
@@ -735,33 +735,43 @@
                        <span class="n">descriptor</span><span class="o">=</span><span class="nb">property</span><span class="p">(</span><span class="n">cls</span><span class="o">.</span><span class="n">get_extra</span><span class="p">,</span> <span class="n">cls</span><span class="o">.</span><span class="n">set_extra</span><span class="p">))</span>
 
     <span class="k">def</span> <span class="nf">get_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
-        <span class="kn">from</span> <span class="nn">airflow</span> <span class="kn">import</span> <span class="n">hooks</span>
-        <span class="kn">from</span> <span class="nn">airflow.contrib</span> <span class="kn">import</span> <span class="n">hooks</span> <span class="k">as</span> <span class="n">contrib_hooks</span>
         <span class="k">try</span><span class="p">:</span>
             <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">&#39;mysql&#39;</span><span class="p">:</span>
-                <span class="k">return</span> <span class="n">hooks</span><span class="o">.</span><span class="n">MySqlHook</span><span class="p">(</span><span class="n">mysql_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+                <span class="kn">from</span> <span class="nn">airflow.hooks.mysql_hook</span> <span class="kn">import</span> <span class="n">MySqlHook</span>
+                <span class="k">return</span> <span class="n">MySqlHook</span><span class="p">(</span><span class="n">mysql_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
             <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">&#39;google_cloud_platform&#39;</span><span class="p">:</span>
-                <span class="k">return</span> <span class="n">contrib_hooks</span><span class="o">.</span><span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+                <span class="kn">from</span> <span class="nn">airflow.contrib.hooks.bigquery_hook</span> <span class="kn">import</span> <span class="n">BigQueryHook</span>
+                <span class="k">return</span> <span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
             <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">&#39;postgres&#39;</span><span class="p">:</span>
-                <span class="k">return</span> <span class="n">hooks</span><span class="o">.</span><span class="n">PostgresHook</span><span class="p">(</span><span class="n">postgres_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+                <span class="kn">from</span> <span class="nn">airflow.hooks.postgres_hook</span> <span class="kn">import</span> <span class="n">PostgresHook</span>
+                <span class="k">return</span> <span class="n">PostgresHook</span><span class="p">(</span><span class="n">postgres_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
             <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">&#39;hive_cli&#39;</span><span class="p">:</span>
-                <span class="k">return</span> <span class="n">hooks</span><span class="o">.</span><span class="n">HiveCliHook</span><span class="p">(</span><span class="n">hive_cli_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+                <span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span>
+                <span class="k">return</span> <span class="n">HiveCliHook</span><span class="p">(</span><span class="n">hive_cli_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
             <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">&#39;presto&#39;</span><span class="p">:</span>
-                <span class="k">return</span> <span class="n">hooks</span><span class="o">.</span><span class="n">PrestoHook</span><span class="p">(</span><span class="n">presto_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+                <span class="kn">from</span> <span class="nn">airflow.hooks.presto_hook</span> <span class="kn">import</span> <span class="n">PrestoHook</span>
+                <span class="k">return</span> <span class="n">PrestoHook</span><span class="p">(</span><span class="n">presto_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
             <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">&#39;hiveserver2&#39;</span><span class="p">:</span>
-                <span class="k">return</span> <span class="n">hooks</span><span class="o">.</span><span class="n">HiveServer2Hook</span><span class="p">(</span><span class="n">hiveserver2_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+                <span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveServer2Hook</span>
+                <span class="k">return</span> <span class="n">HiveServer2Hook</span><span class="p">(</span><span class="n">hiveserver2_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
             <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">&#39;sqlite&#39;</span><span class="p">:</span>
-                <span class="k">return</span> <span class="n">hooks</span><span class="o">.</span><span class="n">SqliteHook</span><span class="p">(</span><span class="n">sqlite_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+                <span class="kn">from</span> <span class="nn">airflow.hooks.sqlite_hook</span> <span class="kn">import</span> <span class="n">SqliteHook</span>
+                <span class="k">return</span> <span class="n">SqliteHook</span><span class="p">(</span><span class="n">sqlite_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
             <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">&#39;jdbc&#39;</span><span class="p">:</span>
-                <span class="k">return</span> <span class="n">hooks</span><span class="o">.</span><span class="n">JdbcHook</span><span class="p">(</span><span class="n">jdbc_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+                <span class="kn">from</span> <span class="nn">airflow.hooks.jdbc_hook</span> <span class="kn">import</span> <span class="n">JdbcHook</span>
+                <span class="k">return</span> <span class="n">JdbcHook</span><span class="p">(</span><span class="n">jdbc_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
             <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">&#39;mssql&#39;</span><span class="p">:</span>
-                <span class="k">return</span> <span class="n">hooks</span><span class="o">.</span><span class="n">MsSqlHook</span><span class="p">(</span><span class="n">mssql_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+                <span class="kn">from</span> <span class="nn">airflow.hooks.mssql_hook</span> <span class="kn">import</span> <span class="n">MsSqlHook</span>
+                <span class="k">return</span> <span class="n">MsSqlHook</span><span class="p">(</span><span class="n">mssql_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
             <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">&#39;oracle&#39;</span><span class="p">:</span>
-                <span class="k">return</span> <span class="n">hooks</span><span class="o">.</span><span class="n">OracleHook</span><span class="p">(</span><span class="n">oracle_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+                <span class="kn">from</span> <span class="nn">airflow.hooks.oracle_hook</span> <span class="kn">import</span> <span class="n">OracleHook</span>
+                <span class="k">return</span> <span class="n">OracleHook</span><span class="p">(</span><span class="n">oracle_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
             <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">&#39;vertica&#39;</span><span class="p">:</span>
-                <span class="k">return</span> <span class="n">contrib_hooks</span><span class="o">.</span><span class="n">VerticaHook</span><span class="p">(</span><span class="n">vertica_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+                <span class="kn">from</span> <span class="nn">airflow.contrib.hooks.vertica_hook</span> <span class="kn">import</span> <span class="n">VerticaHook</span>
+                <span class="k">return</span> <span class="n">VerticaHook</span><span class="p">(</span><span class="n">vertica_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
             <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">&#39;cloudant&#39;</span><span class="p">:</span>
-                <span class="k">return</span> <span class="n">contrib_hooks</span><span class="o">.</span><span class="n">CloudantHook</span><span class="p">(</span><span class="n">cloudant_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+                <span class="kn">from</span> <span class="nn">airflow.contrib.hooks.cloudant_hook</span> <span class="kn">import</span> <span class="n">CloudantHook</span>
+                <span class="k">return</span> <span class="n">CloudantHook</span><span class="p">(</span><span class="n">cloudant_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
         <span class="k">except</span><span class="p">:</span>
             <span class="k">return</span> <span class="bp">None</span>
 
@@ -844,6 +854,7 @@
 
     <span class="n">__table_args__</span> <span class="o">=</span> <span class="p">(</span>
         <span class="n">Index</span><span class="p">(</span><span class="s1">&#39;ti_dag_state&#39;</span><span class="p">,</span> <span class="n">dag_id</span><span class="p">,</span> <span class="n">state</span><span class="p">),</span>
+        <span class="n">Index</span><span class="p">(</span><span class="s1">&#39;ti_state&#39;</span><span class="p">,</span> <span class="n">state</span><span class="p">),</span>
         <span class="n">Index</span><span class="p">(</span><span class="s1">&#39;ti_state_lkp&#39;</span><span class="p">,</span> <span class="n">dag_id</span><span class="p">,</span> <span class="n">task_id</span><span class="p">,</span> <span class="n">execution_date</span><span class="p">,</span> <span class="n">state</span><span class="p">),</span>
         <span class="n">Index</span><span class="p">(</span><span class="s1">&#39;ti_pool&#39;</span><span class="p">,</span> <span class="n">pool</span><span class="p">,</span> <span class="n">state</span><span class="p">,</span> <span class="n">priority_weight</span><span class="p">),</span>
     <span class="p">)</span>
@@ -1308,17 +1319,32 @@
 <div class="viewcode-block" id="TaskInstance.run"><a class="viewcode-back" href="../../code.html#airflow.models.TaskInstance.run">[docs]</a>    <span class="k">def</span> <span class="nf">run</span><span class="p">(</span>
             <span class="bp">self</span><span class="p">,</span>
             <span class="n">verbose</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span>
-            <span class="n">ignore_dependencies</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>  <span class="c1"># Doesn&#39;t check for deps, just runs</span>
-            <span class="n">ignore_depends_on_past</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>   <span class="c1"># Ignore depends_on_past but respect</span>
-                                            <span class="c1"># other deps</span>
-            <span class="n">force</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>  <span class="c1"># Disregards previous successes</span>
-            <span class="n">mark_success</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>  <span class="c1"># Don&#39;t run the task, act as if it succeeded</span>
-            <span class="n">test_mode</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>  <span class="c1"># Doesn&#39;t record success or failure in the DB</span>
+            <span class="n">ignore_dependencies</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
+            <span class="n">ignore_depends_on_past</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
+            <span class="n">force</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
+            <span class="n">mark_success</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
+            <span class="n">test_mode</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
             <span class="n">job_id</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
             <span class="n">pool</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
             <span class="n">session</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Runs the task instance.</span>
+
+<span class="sd">        :param verbose: whether to turn on more verbose loggin</span>
+<span class="sd">        :type verbose: boolean</span>
+<span class="sd">        :param ignore_dependencies: Doesn&#39;t check for deps, just runs</span>
+<span class="sd">        :type ignore_dependencies: boolean</span>
+<span class="sd">        :param ignore_depends_on_past: Ignore depends_on_past but respect</span>
+<span class="sd">            other dependencies</span>
+<span class="sd">        :type ignore_depends_on_past: boolean</span>
+<span class="sd">        :param force: Forces a run regarless of previous success</span>
+<span class="sd">        :type force: boolean</span>
+<span class="sd">        :param mark_success: Don&#39;t run the task, mark its state as success</span>
+<span class="sd">        :type mark_success: boolean</span>
+<span class="sd">        :param test_mode: Doesn&#39;t record success or failure in the DB</span>
+<span class="sd">        :type test_mode: boolean</span>
+<span class="sd">        :param pool: specifies the pool to use to run the task instance</span>
+<span class="sd">        :type pool: str</span>
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="n">task</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">pool</span> <span class="o">=</span> <span class="n">pool</span> <span class="ow">or</span> <span class="n">task</span><span class="o">.</span><span class="n">pool</span>
@@ -1560,6 +1586,24 @@
         <span class="k">if</span> <span class="n">task</span><span class="o">.</span><span class="n">params</span><span class="p">:</span>
             <span class="n">params</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">task</span><span class="o">.</span><span class="n">params</span><span class="p">)</span>
 
+        <span class="k">class</span> <span class="nc">VariableAccessor</span><span class="p">:</span>
+            <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">            Wrapper around Variable. This way you can get variables in templates by using</span>
+<span class="sd">            {var.variable_name}.</span>
+<span class="sd">            &quot;&quot;&quot;</span>
+            <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+                <span class="k">pass</span>
+
+            <span class="k">def</span> <span class="nf">__getattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">item</span><span class="p">):</span>
+                <span class="k">return</span> <span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
+
+        <span class="k">class</span> <span class="nc">VariableJsonAccessor</span><span class="p">:</span>
+            <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+                <span class="k">pass</span>
+
+            <span class="k">def</span> <span class="nf">__getattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">item</span><span class="p">):</span>
+                <span class="k">return</span> <span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="n">deserialize_json</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span>
+
         <span class="k">return</span> <span class="p">{</span>
             <span class="s1">&#39;dag&#39;</span><span class="p">:</span> <span class="n">task</span><span class="o">.</span><span class="n">dag</span><span class="p">,</span>
             <span class="s1">&#39;ds&#39;</span><span class="p">:</span> <span class="n">ds</span><span class="p">,</span>
@@ -1585,6 +1629,10 @@
             <span class="s1">&#39;task_instance_key_str&#39;</span><span class="p">:</span> <span class="n">ti_key_str</span><span class="p">,</span>
             <span class="s1">&#39;conf&#39;</span><span class="p">:</span> <span class="n">configuration</span><span class="p">,</span>
             <span class="s1">&#39;test_mode&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">test_mode</span><span class="p">,</span>
+            <span class="s1">&#39;var&#39;</span><span class="p">:</span> <span class="p">{</span>
+                <span class="s1">&#39;value&#39;</span><span class="p">:</span> <span class="n">VariableAccessor</span><span class="p">(),</span>
+                <span class="s1">&#39;json&#39;</span><span class="p">:</span> <span class="n">VariableJsonAccessor</span><span class="p">()</span>
+            <span class="p">}</span>
         <span class="p">}</span>
 
     <span class="k">def</span> <span class="nf">render_templates</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
@@ -2706,6 +2754,21 @@
         <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_schedule_interval</span><span class="p">,</span> <span class="n">timedelta</span><span class="p">):</span>
             <span class="k">return</span> <span class="n">dttm</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">_schedule_interval</span>
 
+<div class="viewcode-block" id="DAG.normalize_schedule"><a class="viewcode-back" href="../../code.html#airflow.models.DAG.normalize_schedule">[docs]</a>    <span class="k">def</span> <span class="nf">normalize_schedule</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dttm</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Returns dttm + interval unless dttm is first interval then it returns dttm</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">following</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">following_schedule</span><span class="p">(</span><span class="n">dttm</span><span class="p">)</span>
+
+        <span class="c1"># in case of @once</span>
+        <span class="k">if</span> <span class="ow">not</span> <span class="n">following</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">dttm</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">previous_schedule</span><span class="p">(</span><span class="n">following</span><span class="p">)</span> <span class="o">!=</span> <span class="n">dttm</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">following</span>
+
+        <span class="k">return</span> <span class="n">dttm</span></div>
+
     <span class="nd">@property</span>
     <span class="k">def</span> <span class="nf">tasks</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
         <span class="k">return</span> <span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_dict</span><span class="o">.</span><span class="n">values</span><span class="p">())</span>
@@ -3485,10 +3548,12 @@
 
     <span class="nd">@staticmethod</span>
     <span class="nd">@provide_session</span>
-    <span class="k">def</span> <span class="nf">find</span><span class="p">(</span><span class="n">dag_id</span><span class="p">,</span> <span class="n">run_id</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">execution_date</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+    <span class="k">def</span> <span class="nf">find</span><span class="p">(</span><span class="n">dag_id</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">run_id</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">execution_date</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
              <span class="n">state</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">external_trigger</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Returns a set of dag runs for the given search criteria.</span>
+<span class="sd">        :param dag_id: the dag_id to find dag runs for</span>
+<span class="sd">        :type dag_id: integer, list</span>
 <span class="sd">        :param run_id: defines the the run id for this dag run</span>
 <span class="sd">        :type run_id: string</span>
 <span class="sd">        :param execution_date: the execution date</span>
@@ -3502,7 +3567,9 @@
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="n">DR</span> <span class="o">=</span> <span class="n">DagRun</span>
 
-        <span class="n">qry</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DR</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DR</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="n">dag_id</span><span class="p">)</span>
+        <span class="n">qry</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DR</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">dag_id</span><span class="p">:</span>
+            <span class="n">qry</span> <span class="o">=</span> <span class="n">qry</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DR</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="n">dag_id</span><span class="p">)</span>
         <span class="k">if</span> <span class="n">run_id</span><span class="p">:</span>
             <span class="n">qry</span> <span class="o">=</span> <span class="n">qry</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DR</span><span class="o">.</span><span class="n">run_id</span> <span class="o">==</span> <span class="n">run_id</span><span class="p">)</span>
         <span class="k">if</span> <span class="n">execution_date</span><span class="p">:</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/airflow/operators/docker_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/docker_operator.html b/_modules/airflow/operators/docker_operator.html
index 46fcf7b..06aa65e 100644
--- a/_modules/airflow/operators/docker_operator.html
+++ b/_modules/airflow/operators/docker_operator.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for airflow.operators.docker_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">json</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">json</span>
 <span class="kn">import</span> <span class="nn">logging</span>
 <span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>

[2/5] incubator-airflow-site git commit: Updating the disclaimer

[ma...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/slack_operator.html
----------------------------------------------------------------------
diff --git a/_modules/slack_operator.html b/_modules/slack_operator.html
index f958c5b..152902b 100644
--- a/_modules/slack_operator.html
+++ b/_modules/slack_operator.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for slack_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">slackclient</span> <span class="kn">import</span> <span class="n">SlackClient</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">slackclient</span> <span class="kn">import</span> <span class="n">SlackClient</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 <span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/sqlite_hook.html
----------------------------------------------------------------------
diff --git a/_modules/sqlite_hook.html b/_modules/sqlite_hook.html
index 6ed5de8..70abb8c 100644
--- a/_modules/sqlite_hook.html
+++ b/_modules/sqlite_hook.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for sqlite_hook</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">sqlite3</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">sqlite3</span>
 
 <span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="kn">import</span> <span class="n">DbApiHook</span>
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/ssh_execute_operator.html
----------------------------------------------------------------------
diff --git a/_modules/ssh_execute_operator.html b/_modules/ssh_execute_operator.html
index 4346f31..c4c952d 100644
--- a/_modules/ssh_execute_operator.html
+++ b/_modules/ssh_execute_operator.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for ssh_execute_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">bytes</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">bytes</span>
 <span class="kn">import</span> <span class="nn">logging</span>
 <span class="kn">import</span> <span class="nn">subprocess</span>
 <span class="kn">from</span> <span class="nn">subprocess</span> <span class="kn">import</span> <span class="n">STDOUT</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/vertica_operator.html
----------------------------------------------------------------------
diff --git a/_modules/vertica_operator.html b/_modules/vertica_operator.html
index 2ae2681..46f8a92 100644
--- a/_modules/vertica_operator.html
+++ b/_modules/vertica_operator.html
@@ -137,9 +137,23 @@
            <div itemprop="articleBody">
             
   <h1>Source code for vertica_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
-
-<span class="kn">from</span> <span class="nn">airflow.contrib.hooks</span> <span class="kn">import</span> <span class="n">VerticaHook</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.vertica_hook</span> <span class="kn">import</span> <span class="n">VerticaHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/vertica_to_hive.html
----------------------------------------------------------------------
diff --git a/_modules/vertica_to_hive.html b/_modules/vertica_to_hive.html
index 57c83f1..a700374 100644
--- a/_modules/vertica_to_hive.html
+++ b/_modules/vertica_to_hive.html
@@ -137,14 +137,28 @@
            <div itemprop="articleBody">
             
   <h1>Source code for vertica_to_hive</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">chr</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">chr</span>
 <span class="kn">from</span> <span class="nn">collections</span> <span class="kn">import</span> <span class="n">OrderedDict</span>
 <span class="kn">import</span> <span class="nn">unicodecsv</span> <span class="kn">as</span> <span class="nn">csv</span>
 <span class="kn">import</span> <span class="nn">logging</span>
 <span class="kn">from</span> <span class="nn">tempfile</span> <span class="kn">import</span> <span class="n">NamedTemporaryFile</span>
 
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span>
-<span class="kn">from</span> <span class="nn">airflow.contrib.hooks</span> <span class="kn">import</span> <span class="n">VerticaHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.vertica_hook</span> <span class="kn">import</span> <span class="n">VerticaHook</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
 <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/webhdfs_hook.html
----------------------------------------------------------------------
diff --git a/_modules/webhdfs_hook.html b/_modules/webhdfs_hook.html
index d54cb05..3a44028 100644
--- a/_modules/webhdfs_hook.html
+++ b/_modules/webhdfs_hook.html
@@ -137,7 +137,21 @@
            <div itemprop="articleBody">
             
   <h1>Source code for webhdfs_hook</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="kn">import</span> <span class="n">BaseHook</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="kn">import</span> <span class="n">BaseHook</span>
 <span class="kn">from</span> <span class="nn">airflow</span> <span class="kn">import</span> <span class="n">configuration</span>
 <span class="kn">import</span> <span class="nn">logging</span>
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_sources/code.txt
----------------------------------------------------------------------
diff --git a/_sources/code.txt b/_sources/code.txt
index 2876876..ba1cddb 100644
--- a/_sources/code.txt
+++ b/_sources/code.txt
@@ -133,6 +133,10 @@ Variable                            Description
 ``{{ latest_date }}``               same as ``{{ ds }}``
 ``{{ ti }}``                        same as ``{{ task_instance }}``
 ``{{ params }}``                    a reference to the user-defined params dictionary
+``{{ var.value.my_var }}``          global defined variables represented as a dictionary
+``{{ var.json.my_var.path }}``      global defined variables represented as a dictionary
+                                    with deserialized JSON object, append the path to the
+                                    key within the JSON object
 ``{{ task_instance_key_str }}``     a unique, human-readable key to the task instance
                                     formatted ``{dag_id}_{task_id}_{ds}``
 ``conf``                            the full configuration object located at
@@ -151,6 +155,11 @@ dot notation. Here are some examples of what is possible:
 Refer to the models documentation for more information on the objects'
 attributes and methods.
 
+The ``var`` template variable allows you to access variables defined in Airflow's
+UI. You can access them as either plain-text or JSON. If you use JSON, you are
+also able to walk nested structures, such as dictionaries like:
+``{{ var.json.my_dict_var.key1 }}``
+
 Macros
 ''''''
 Macros are a way to expose objects to your templates and live under the

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_sources/concepts.txt
----------------------------------------------------------------------
diff --git a/_sources/concepts.txt b/_sources/concepts.txt
index 811130d..8cfc8ab 100644
--- a/_sources/concepts.txt
+++ b/_sources/concepts.txt
@@ -485,7 +485,7 @@ the main UI. For example:
 
   #dags/subdag.py
   from airflow.models import DAG
-  from airflow.operators import DummyOperator
+  from airflow.operators.dummy_operator import DummyOperator
 
 
   # Dag is returned by a factory method
@@ -510,7 +510,7 @@ This SubDAG can then be referenced in your main DAG file:
   # main_dag.py
   from datetime import datetime, timedelta
   from airflow.models import DAG
-  from airflow.operators import SubDagOperator
+  from airflow.operators.subdag_operator import SubDagOperator
   from dags.subdag import sub_dag
 
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_sources/faq.txt
----------------------------------------------------------------------
diff --git a/_sources/faq.txt b/_sources/faq.txt
index 21623fc..e61c1bf 100644
--- a/_sources/faq.txt
+++ b/_sources/faq.txt
@@ -78,12 +78,17 @@ We recommend against using dynamic values as ``start_date``, especially
 once the period closes, and in theory an ``@hourly`` DAG would never get to
 an hour after now as ``now()`` moves along.
 
-We also recommend using rounded ``start_date`` in relation to your
-``schedule_interval``. This means an ``@hourly`` would be at ``00:00``
+
+Previously we also recommended using rounded ``start_date`` in relation to your
+``schedule_interval``. This meant an ``@hourly`` would be at ``00:00``
 minutes:seconds, a ``@daily`` job at midnight, a ``@monthly`` job on the
-first of the month. You can use any sensor or a ``TimeDeltaSensor`` to delay
-the execution of tasks within that period. While ``schedule_interval``
-does allow specifying a ``datetime.timedelta``
+first of the month. This is no longer required. Airflow will not auto align
+the ``start_date`` and the ``schedule_interval``, by using the ``start_date``
+as the moment to start looking.
+
+You can use any sensor or a ``TimeDeltaSensor`` to delay
+the execution of tasks within the schedule interval.
+While ``schedule_interval`` does allow specifying a ``datetime.timedelta``
 object, we recommend using the macros or cron expressions instead, as
 it enforces this idea of rounded schedules.
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_sources/index.txt
----------------------------------------------------------------------
diff --git a/_sources/index.txt b/_sources/index.txt
index c98d619..133d5ab 100644
--- a/_sources/index.txt
+++ b/_sources/index.txt
@@ -7,10 +7,10 @@
 Apache Airflow (incubating) Documentation
 =========================================
 
-.. note ::
+.. important::
 
-    Apache Airflow is an effort undergoing incubation at The Apache
-    Software Foundation (ASF), sponsored by Chris Riccomini.
+    **Disclaimer**: Apache Airflow is an effort undergoing incubation at The
+    Apache Software Foundation (ASF), sponsored by the Apache Incubator.
     Incubation is required of all newly accepted projects until a further
     review indicates that the infrastructure, communications, and
     decision making process have stabilized in a manner consistent with
@@ -19,6 +19,7 @@ Apache Airflow (incubating) Documentation
     the code, it does indicate that the project has yet to be fully
     endorsed by the ASF.
 
+
 Airflow is a platform to programmatically author, schedule and monitor
 workflows.
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_sources/tutorial.txt
----------------------------------------------------------------------
diff --git a/_sources/tutorial.txt b/_sources/tutorial.txt
index e9d382b..a93479c 100644
--- a/_sources/tutorial.txt
+++ b/_sources/tutorial.txt
@@ -18,7 +18,7 @@ complicated, a line by line explanation follows below.
     https://github.com/airbnb/airflow/blob/master/airflow/example_dags/tutorial.py
     """
     from airflow import DAG
-    from airflow.operators import BashOperator
+    from airflow.operators.bash_operator import BashOperator
     from datetime import datetime, timedelta
 
 
@@ -100,7 +100,7 @@ Airflow DAG object. Let's start by importing the libraries we will need.
     from airflow import DAG
 
     # Operators; we need this to operate!
-    from airflow.operators import BashOperator
+    from airflow.operators.bash_operator import BashOperator
 
 Default Arguments
 -----------------
@@ -270,7 +270,7 @@ something like this:
     http://airflow.readthedocs.org/en/latest/tutorial.html
     """
     from airflow import DAG
-    from airflow.operators import BashOperator
+    from airflow.operators.bash_operator import BashOperator
     from datetime import datetime, timedelta
 
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/code.html
----------------------------------------------------------------------
diff --git a/code.html b/code.html
index 06a0aab..592ec27 100644
--- a/code.html
+++ b/code.html
@@ -322,6 +322,15 @@ between each tries</li>
 </div>
 <div class="section" id="module-airflow.operators">
 <span id="operator-api"></span><h3>Operator API<a class="headerlink" href="#module-airflow.operators" title="Permalink to this headline">�</a></h3>
+<p>Importer that dynamically loads a class and module from its parent. This
+allows Airflow to support <cite>from airflow.operators.bash_operator import
+BashOperator</cite> even though BashOperator is actually in
+airflow.operators.bash_operator.</p>
+<p>The importer also takes over for the parent_module by wrapping it. This is
+required to support attribute-based usage:</p>
+<blockquote>
+<div>from airflow import operators
+operators.BashOperator(...)</div></blockquote>
 <dl class="class">
 <dt id="airflow.operators.BashOperator">
 <em class="property">class </em><code class="descclassname">airflow.operators.</code><code class="descname">BashOperator</code><span class="sig-paren">(</span><em>bash_command</em>, <em>xcom_push=False</em>, <em>env=None</em>, <em>output_encoding='utf-8'</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/bash_operator.html#BashOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.operators.BashOperator" title="Permalink to this definition">�</a></dt>
@@ -431,7 +440,7 @@ is allowed</li>
 
 <dl class="class">
 <dt id="airflow.operators.ExternalTaskSensor">
-<em class="property">class </em><code class="descclassname">airflow.operators.</code><code class="descname">ExternalTaskSensor</code><span class="sig-paren">(</span><em>external_dag_id</em>, <em>external_task_id</em>, <em>allowed_states=None</em>, <em>execution_delta=None</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/sensors.html#ExternalTaskSensor"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.operators.ExternalTaskSensor" title="Permalink to this definition">�</a></dt>
+<em class="property">class </em><code class="descclassname">airflow.operators.</code><code class="descname">ExternalTaskSensor</code><span class="sig-paren">(</span><em>external_dag_id</em>, <em>external_task_id</em>, <em>allowed_states=None</em>, <em>execution_delta=None</em>, <em>execution_date_fn=None</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/sensors.html#ExternalTaskSensor"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.operators.ExternalTaskSensor" title="Permalink to this definition">�</a></dt>
 <dd><p>Bases: <a class="reference internal" href="#airflow.operators.sensors.BaseSensorOperator" title="airflow.operators.sensors.BaseSensorOperator"><code class="xref py py-class docutils literal"><span class="pre">sensors.BaseSensorOperator</span></code></a></p>
 <p>Waits for a task to complete in a different DAG</p>
 <table class="docutils field-list" frame="void" rules="none">
@@ -446,7 +455,12 @@ wait for</li>
 <li><strong>allowed_states</strong> (<em>list</em>) &#8211; list of allowed states, default is <code class="docutils literal"><span class="pre">['success']</span></code></li>
 <li><strong>execution_delta</strong> (<em>datetime.timedelta</em>) &#8211; time difference with the previous execution to
 look at, the default is the same execution_date as the current task.
-For yesterday, use [positive!] datetime.timedelta(days=1)</li>
+For yesterday, use [positive!] datetime.timedelta(days=1). Either
+execution_delta or execution_date_fn can be passed to
+ExternalTaskSensor, but not both.</li>
+<li><strong>execution_date_fn</strong> (<em>callable</em>) &#8211; function that receives the current execution date
+and returns the desired execution date to query. Either execution_delta
+or execution_date_fn can be passed to ExternalTaskSensor, but not both.</li>
 </ul>
 </td>
 </tr>
@@ -539,7 +553,10 @@ part of the script before the first occurrence of <cite>script_begin_tag</cite><
 <dt id="airflow.operators.HivePartitionSensor">
 <em class="property">class </em><code class="descclassname">airflow.operators.</code><code class="descname">HivePartitionSensor</code><span class="sig-paren">(</span><em>table</em>, <em>partition=&quot;ds='{{ ds }}'&quot;</em>, <em>metastore_conn_id='metastore_default'</em>, <em>schema='default'</em>, <em>poke_interval=180</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/sensors.html#HivePartitionSensor"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.operators.HivePartitionSensor" title="Permalink to this definition">�</a></dt>
 <dd><p>Bases: <a class="reference internal" href="#airflow.operators.sensors.BaseSensorOperator" title="airflow.operators.sensors.BaseSensorOperator"><code class="xref py py-class docutils literal"><span class="pre">sensors.BaseSensorOperator</span></code></a></p>
-<p>Waits for a partition to show up in Hive</p>
+<p>Waits for a partition to show up in Hive.</p>
+<p>Note: Because &#64;partition supports general logical operators, it
+can be inefficient. Consider using NamedHivePartitionSensor instead if
+you don&#8217;t need the full flexibility of HivePartitionSensor.</p>
 <table class="docutils field-list" frame="void" rules="none">
 <col class="field-name" />
 <col class="field-body" />
@@ -548,7 +565,7 @@ part of the script before the first occurrence of <cite>script_begin_tag</cite><
 <li><strong>table</strong> (<em>string</em>) &#8211; The name of the table to wait for, supports the dot
 notation (my_database.my_table)</li>
 <li><strong>partition</strong> (<em>string</em>) &#8211; The partition clause to wait for. This is passed as
-is to the Metastore Thrift client &#8220;get_partitions_by_filter&#8221; method,
+is to the metastore Thrift client &#8220;get_partitions_by_filter&#8221; method,
 and apparently supports SQL like notation as in <cite>ds=&#8216;2015-01-01&#8217;
 AND type=&#8217;value&#8217;</cite> and &gt; &lt; sings as in &#8220;ds&gt;=2015-01-01&#8221;</li>
 <li><strong>metastore_conn_id</strong> (<em>str</em>) &#8211; reference to the metastore thrift service
@@ -841,7 +858,7 @@ Template reference are recognized by str ending in '.sql'</em>) &#8211; the sql
 <dl class="class">
 <dt id="airflow.operators.PrestoCheckOperator">
 <em class="property">class </em><code class="descclassname">airflow.operators.</code><code class="descname">PrestoCheckOperator</code><span class="sig-paren">(</span><em>sql</em>, <em>presto_conn_id='presto_default'</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/presto_check_operator.html#PrestoCheckOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.operators.PrestoCheckOperator" title="Permalink to this definition">�</a></dt>
-<dd><p>Bases: <code class="xref py py-class docutils literal"><span class="pre">check_operator.CheckOperator</span></code></p>
+<dd><p>Bases: <code class="xref py py-class docutils literal"><span class="pre">airflow.operators.check_operator.CheckOperator</span></code></p>
 <p>Performs checks against Presto. The <code class="docutils literal"><span class="pre">PrestoCheckOperator</span></code> expects
 a sql query that will return a single row. Each value on that
 first row is evaluated using python <code class="docutils literal"><span class="pre">bool</span></code> casting. If any of the
@@ -880,7 +897,7 @@ without stopping the progress of the DAG.</p>
 <dl class="class">
 <dt id="airflow.operators.PrestoIntervalCheckOperator">
 <em class="property">class </em><code class="descclassname">airflow.operators.</code><code class="descname">PrestoIntervalCheckOperator</code><span class="sig-paren">(</span><em>table</em>, <em>metrics_thresholds</em>, <em>date_filter_column='ds'</em>, <em>days_back=-7</em>, <em>presto_conn_id='presto_default'</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/presto_check_operator.html#PrestoIntervalCheckOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.operators.PrestoIntervalCheckOperator" title="Permalink to this definition">�</a></dt>
-<dd><p>Bases: <code class="xref py py-class docutils literal"><span class="pre">check_operator.IntervalCheckOperator</span></code></p>
+<dd><p>Bases: <code class="xref py py-class docutils literal"><span class="pre">airflow.operators.check_operator.IntervalCheckOperator</span></code></p>
 <p>Checks that the values of metrics given as SQL expressions are within
 a certain tolerance of the ones from days_back before.</p>
 <table class="docutils field-list" frame="void" rules="none">
@@ -903,7 +920,7 @@ against. Defaults to 7 days</li>
 <dl class="class">
 <dt id="airflow.operators.PrestoValueCheckOperator">
 <em class="property">class </em><code class="descclassname">airflow.operators.</code><code class="descname">PrestoValueCheckOperator</code><span class="sig-paren">(</span><em>sql</em>, <em>pass_value</em>, <em>tolerance=None</em>, <em>presto_conn_id='presto_default'</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/presto_check_operator.html#PrestoValueCheckOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.operators.PrestoValueCheckOperator" title="Permalink to this definition">�</a></dt>
-<dd><p>Bases: <code class="xref py py-class docutils literal"><span class="pre">check_operator.ValueCheckOperator</span></code></p>
+<dd><p>Bases: <code class="xref py py-class docutils literal"><span class="pre">airflow.operators.check_operator.ValueCheckOperator</span></code></p>
 <p>Performs a simple value check using sql code.</p>
 <table class="docutils field-list" frame="void" rules="none">
 <col class="field-name" />
@@ -1188,6 +1205,15 @@ The default is False.</li>
 </div>
 <div class="section" id="module-airflow.contrib.operators">
 <span id="community-contributed-operators"></span><h3>Community-contributed Operators<a class="headerlink" href="#module-airflow.contrib.operators" title="Permalink to this headline">�</a></h3>
+<p>Importer that dynamically loads a class and module from its parent. This
+allows Airflow to support <cite>from airflow.operators.bash_operator import
+BashOperator</cite> even though BashOperator is actually in
+airflow.operators.bash_operator.</p>
+<p>The importer also takes over for the parent_module by wrapping it. This is
+required to support attribute-based usage:</p>
+<blockquote>
+<div>from airflow import operators
+operators.BashOperator(...)</div></blockquote>
 <dl class="class">
 <dt id="airflow.contrib.operators.SSHExecuteOperator">
 <em class="property">class </em><code class="descclassname">airflow.contrib.operators.</code><code class="descname">SSHExecuteOperator</code><span class="sig-paren">(</span><em>ssh_hook</em>, <em>bash_command</em>, <em>xcom_push=False</em>, <em>env=None</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/ssh_execute_operator.html#SSHExecuteOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.SSHExecuteOperator" title="Permalink to this definition">�</a></dt>
@@ -1286,7 +1312,7 @@ specific database</li>
 
 <dl class="class">
 <dt id="airflow.contrib.operators.gcs_download_operator.GoogleCloudStorageDownloadOperator">
-<em class="property">class </em><code class="descclassname">airflow.contrib.operators.gcs_download_operator.</code><code class="descname">GoogleCloudStorageDownloadOperator</code><span class="sig-paren">(</span><em>bucket</em>, <em>object</em>, <em>filename</em>, <em>google_cloud_storage_conn_id='google_cloud_storage_default'</em>, <em>delegate_to=None</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/contrib/operators/gcs_download_operator.html#GoogleCloudStorageDownloadOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.gcs_download_operator.GoogleCloudStorageDownloadOperator" title="Permalink to this definition">�</a></dt>
+<em class="property">class </em><code class="descclassname">airflow.contrib.operators.gcs_download_operator.</code><code class="descname">GoogleCloudStorageDownloadOperator</code><span class="sig-paren">(</span><em>bucket</em>, <em>object</em>, <em>filename=False</em>, <em>store_to_xcom_key=False</em>, <em>google_cloud_storage_conn_id='google_cloud_storage_default'</em>, <em>delegate_to=None</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/contrib/operators/gcs_download_operator.html#GoogleCloudStorageDownloadOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.gcs_download_operator.GoogleCloudStorageDownloadOperator" title="Permalink to this definition">�</a></dt>
 <dd><p>Downloads a file from Google Cloud Storage.</p>
 </dd></dl>
 
@@ -1560,8 +1586,8 @@ More info: <a class="reference external" href="https://www.hipchat.com/docs/apiv
 in all templates</p>
 <table border="1" class="docutils">
 <colgroup>
-<col width="40%" />
-<col width="60%" />
+<col width="38%" />
+<col width="62%" />
 </colgroup>
 <thead valign="bottom">
 <tr class="row-odd"><th class="head">Variable</th>
@@ -1620,6 +1646,14 @@ in all templates</p>
 <tr class="row-even"><td><code class="docutils literal"><span class="pre">{{</span> <span class="pre">params</span> <span class="pre">}}</span></code></td>
 <td>a reference to the user-defined params dictionary</td>
 </tr>
+<tr class="row-odd"><td><code class="docutils literal"><span class="pre">{{</span> <span class="pre">var.value.my_var</span> <span class="pre">}}</span></code></td>
+<td>global defined variables represented as a dictionary</td>
+</tr>
+<tr class="row-even"><td><code class="docutils literal"><span class="pre">{{</span> <span class="pre">var.json.my_var.path</span> <span class="pre">}}</span></code></td>
+<td>global defined variables represented as a dictionary
+with deserialized JSON object, append the path to the
+key within the JSON object</td>
+</tr>
 <tr class="row-odd"><td><code class="docutils literal"><span class="pre">{{</span> <span class="pre">task_instance_key_str</span> <span class="pre">}}</span></code></td>
 <td>a unique, human-readable key to the task instance
 formatted <code class="docutils literal"><span class="pre">{dag_id}_{task_id}_{ds}</span></code></td>
@@ -1647,6 +1681,10 @@ dot notation. Here are some examples of what is possible:
 <code class="docutils literal"><span class="pre">{{</span> <span class="pre">task.owner</span> <span class="pre">}}</span></code>, <code class="docutils literal"><span class="pre">{{</span> <span class="pre">task.task_id</span> <span class="pre">}}</span></code>, <code class="docutils literal"><span class="pre">{{</span> <span class="pre">ti.hostname</span> <span class="pre">}}</span></code>, ...
 Refer to the models documentation for more information on the objects&#8217;
 attributes and methods.</p>
+<p>The <code class="docutils literal"><span class="pre">var</span></code> template variable allows you to access variables defined in Airflow&#8217;s
+UI. You can access them as either plain-text or JSON. If you use JSON, you are
+also able to walk nested structures, such as dictionaries like:
+<code class="docutils literal"><span class="pre">{{</span> <span class="pre">var.json.my_dict_var.key1</span> <span class="pre">}}</span></code></p>
 </div>
 <div class="section" id="id2">
 <h3>Macros<a class="headerlink" href="#id2" title="Permalink to this headline">�</a></h3>
@@ -1736,12 +1774,6 @@ as specified in the output format</p>
 </dd></dl>
 
 <dl class="function">
-<dt id="airflow.macros.integrate_plugins">
-<code class="descclassname">airflow.macros.</code><code class="descname">integrate_plugins</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/macros.html#integrate_plugins"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.macros.integrate_plugins" title="Permalink to this definition">�</a></dt>
-<dd><p>Integrate plugins to the context</p>
-</dd></dl>
-
-<dl class="function">
 <dt id="airflow.macros.random">
 <code class="descclassname">airflow.macros.</code><code class="descname">random</code><span class="sig-paren">(</span><span class="sig-paren">)</span> &rarr; x in the interval [0, 1).<a class="headerlink" href="#airflow.macros.random" title="Permalink to this definition">�</a></dt>
 <dd></dd></dl>
@@ -1972,6 +2004,12 @@ template_searchpath and user_defined_macros</p>
 </dd></dl>
 
 <dl class="method">
+<dt id="airflow.models.DAG.normalize_schedule">
+<code class="descname">normalize_schedule</code><span class="sig-paren">(</span><em>dttm</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/models.html#DAG.normalize_schedule"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.DAG.normalize_schedule" title="Permalink to this definition">�</a></dt>
+<dd><p>Returns dttm + interval unless dttm is first interval then it returns dttm</p>
+</dd></dl>
+
+<dl class="method">
 <dt id="airflow.models.DAG.run">
 <code class="descname">run</code><span class="sig-paren">(</span><em>start_date=None</em>, <em>end_date=None</em>, <em>mark_success=False</em>, <em>include_adhoc=False</em>, <em>local=False</em>, <em>executor=None</em>, <em>donot_pickle=False</em>, <em>ignore_dependencies=False</em>, <em>ignore_first_depends_on_past=False</em>, <em>pool=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/models.html#DAG.run"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.DAG.run" title="Permalink to this definition">�</a></dt>
 <dd><p>Runs the DAG.</p>
@@ -2465,6 +2503,24 @@ session is committed.</td>
 <dt id="airflow.models.TaskInstance.run">
 <code class="descname">run</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/models.html#TaskInstance.run"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.TaskInstance.run" title="Permalink to this definition">�</a></dt>
 <dd><p>Runs the task instance.</p>
+<table class="docutils field-list" frame="void" rules="none">
+<col class="field-name" />
+<col class="field-body" />
+<tbody valign="top">
+<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple">
+<li><strong>verbose</strong> (<em>boolean</em>) &#8211; whether to turn on more verbose loggin</li>
+<li><strong>ignore_dependencies</strong> (<em>boolean</em>) &#8211; Doesn&#8217;t check for deps, just runs</li>
+<li><strong>ignore_depends_on_past</strong> (<em>boolean</em>) &#8211; Ignore depends_on_past but respect
+other dependencies</li>
+<li><strong>force</strong> (<em>boolean</em>) &#8211; Forces a run regarless of previous success</li>
+<li><strong>mark_success</strong> (<em>boolean</em>) &#8211; Don&#8217;t run the task, mark its state as success</li>
+<li><strong>test_mode</strong> (<em>boolean</em>) &#8211; Doesn&#8217;t record success or failure in the DB</li>
+<li><strong>pool</strong> (<em>str</em>) &#8211; specifies the pool to use to run the task instance</li>
+</ul>
+</td>
+</tr>
+</tbody>
+</table>
 </dd></dl>
 
 <dl class="method">
@@ -2631,6 +2687,15 @@ passwords when using operators or hooks.</p>
 </div>
 <div class="section" id="module-airflow.hooks">
 <span id="hooks"></span><h2>Hooks<a class="headerlink" href="#module-airflow.hooks" title="Permalink to this headline">�</a></h2>
+<p>Importer that dynamically loads a class and module from its parent. This
+allows Airflow to support <cite>from airflow.operators.bash_operator import
+BashOperator</cite> even though BashOperator is actually in
+airflow.operators.bash_operator.</p>
+<p>The importer also takes over for the parent_module by wrapping it. This is
+required to support attribute-based usage:</p>
+<blockquote>
+<div>from airflow import operators
+operators.BashOperator(...)</div></blockquote>
 <dl class="class">
 <dt id="airflow.hooks.DbApiHook">
 <em class="property">class </em><code class="descclassname">airflow.hooks.</code><code class="descname">DbApiHook</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/dbapi_hook.html#DbApiHook"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.hooks.DbApiHook" title="Permalink to this definition">�</a></dt>
@@ -2799,7 +2864,8 @@ traditional CLI. To enable <code class="docutils literal"><span class="pre">beel
 extra field of your connection as in <code class="docutils literal"><span class="pre">{</span> <span class="pre">&quot;use_beeline&quot;:</span> <span class="pre">true</span> <span class="pre">}</span></code></p>
 <p>Note that you can also set default hive CLI parameters using the
 <code class="docutils literal"><span class="pre">hive_cli_params</span></code> to be used in your connection as in
-<code class="docutils literal"><span class="pre">{&quot;hive_cli_params&quot;:</span> <span class="pre">&quot;-hiveconf</span> <span class="pre">mapred.job.tracker=some.jobtracker:444&quot;}</span></code></p>
+<code class="docutils literal"><span class="pre">{&quot;hive_cli_params&quot;:</span> <span class="pre">&quot;-hiveconf</span> <span class="pre">mapred.job.tracker=some.jobtracker:444&quot;}</span></code>
+Parameters passed here can be overridden by run_cli&#8217;s hive_conf param</p>
 <p>The extra connection parameter <code class="docutils literal"><span class="pre">auth</span></code> gets passed as in the <code class="docutils literal"><span class="pre">jdbc</span></code>
 connection string as is.</p>
 <dl class="method">
@@ -2834,8 +2900,20 @@ and values</li>
 
 <dl class="method">
 <dt id="airflow.hooks.HiveCliHook.run_cli">
-<code class="descname">run_cli</code><span class="sig-paren">(</span><em>hql</em>, <em>schema=None</em>, <em>verbose=True</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/hive_hooks.html#HiveCliHook.run_cli"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.hooks.HiveCliHook.run_cli" title="Permalink to this definition">�</a></dt>
-<dd><p>Run an hql statement using the hive cli</p>
+<code class="descname">run_cli</code><span class="sig-paren">(</span><em>hql</em>, <em>schema=None</em>, <em>verbose=True</em>, <em>hive_conf=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/hive_hooks.html#HiveCliHook.run_cli"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.hooks.HiveCliHook.run_cli" title="Permalink to this definition">�</a></dt>
+<dd><p>Run an hql statement using the hive cli. If hive_conf is specified it should be a
+dict and the entries will be set as key/value pairs in HiveConf</p>
+<table class="docutils field-list" frame="void" rules="none">
+<col class="field-name" />
+<col class="field-body" />
+<tbody valign="top">
+<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>hive_conf</strong> (<em>dict</em>) &#8211; if specified these key value pairs will be passed to hive as
+<code class="docutils literal"><span class="pre">-hiveconf</span> <span class="pre">&quot;key&quot;=&quot;value&quot;</span></code>. Note that they will be passed after the
+<code class="docutils literal"><span class="pre">hive_cli_params</span></code> and thus will override whatever values are specified in
+the database.</td>
+</tr>
+</tbody>
+</table>
 <div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">hh</span> <span class="o">=</span> <span class="n">HiveCliHook</span><span class="p">()</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">result</span> <span class="o">=</span> <span class="n">hh</span><span class="o">.</span><span class="n">run_cli</span><span class="p">(</span><span class="s2">&quot;USE airflow;&quot;</span><span class="p">)</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="p">(</span><span class="s2">&quot;OK&quot;</span> <span class="ow">in</span> <span class="n">result</span><span class="p">)</span>
@@ -2858,9 +2936,60 @@ and values</li>
 <dd><p>Bases: <code class="xref py py-class docutils literal"><span class="pre">airflow.hooks.base_hook.BaseHook</span></code></p>
 <p>Wrapper to interact with the Hive Metastore</p>
 <dl class="method">
+<dt id="airflow.hooks.HiveMetastoreHook.check_for_named_partition">
+<code class="descname">check_for_named_partition</code><span class="sig-paren">(</span><em>schema</em>, <em>table</em>, <em>partition_name</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/hive_hooks.html#HiveMetastoreHook.check_for_named_partition"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.hooks.HiveMetastoreHook.check_for_named_partition" title="Permalink to this definition">�</a></dt>
+<dd><p>Checks whether a partition with a given name exists</p>
+<table class="docutils field-list" frame="void" rules="none">
+<col class="field-name" />
+<col class="field-body" />
+<tbody valign="top">
+<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
+<li><strong>schema</strong> (<em>string</em>) &#8211; Name of hive schema (database) &#64;table belongs to</li>
+<li><strong>table</strong> &#8211; Name of hive table &#64;partition belongs to</li>
+</ul>
+</td>
+</tr>
+<tr class="field-even field"><th class="field-name">Partition:</th><td class="field-body"><p class="first">Name of the partitions to check for (eg <cite>a=b/c=d</cite>)</p>
+</td>
+</tr>
+<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body"><p class="first last">boolean</p>
+</td>
+</tr>
+</tbody>
+</table>
+<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">hh</span> <span class="o">=</span> <span class="n">HiveMetastoreHook</span><span class="p">()</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">t</span> <span class="o">=</span> <span class="s1">&#39;static_babynames_partitioned&#39;</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">hh</span><span class="o">.</span><span class="n">check_for_named_partition</span><span class="p">(</span><span class="s1">&#39;airflow&#39;</span><span class="p">,</span> <span class="n">t</span><span class="p">,</span> <span class="s2">&quot;ds=2015-01-01&quot;</span><span class="p">)</span>
+<span class="go">True</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">hh</span><span class="o">.</span><span class="n">check_for_named_partition</span><span class="p">(</span><span class="s1">&#39;airflow&#39;</span><span class="p">,</span> <span class="n">t</span><span class="p">,</span> <span class="s2">&quot;ds=xxx&quot;</span><span class="p">)</span>
+<span class="go">False</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="method">
 <dt id="airflow.hooks.HiveMetastoreHook.check_for_partition">
 <code class="descname">check_for_partition</code><span class="sig-paren">(</span><em>schema</em>, <em>table</em>, <em>partition</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/hive_hooks.html#HiveMetastoreHook.check_for_partition"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.hooks.HiveMetastoreHook.check_for_partition" title="Permalink to this definition">�</a></dt>
 <dd><p>Checks whether a partition exists</p>
+<table class="docutils field-list" frame="void" rules="none">
+<col class="field-name" />
+<col class="field-body" />
+<tbody valign="top">
+<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
+<li><strong>schema</strong> (<em>string</em>) &#8211; Name of hive schema (database) &#64;table belongs to</li>
+<li><strong>table</strong> &#8211; Name of hive table &#64;partition belongs to</li>
+</ul>
+</td>
+</tr>
+<tr class="field-even field"><th class="field-name">Partition:</th><td class="field-body"><p class="first">Expression that matches the partitions to check for
+(eg <cite>a = &#8216;b&#8217; AND c = &#8216;d&#8217;</cite>)</p>
+</td>
+</tr>
+<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body"><p class="first last">boolean</p>
+</td>
+</tr>
+</tbody>
+</table>
 <div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">hh</span> <span class="o">=</span> <span class="n">HiveMetastoreHook</span><span class="p">()</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">t</span> <span class="o">=</span> <span class="s1">&#39;static_babynames_partitioned&#39;</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">hh</span><span class="o">.</span><span class="n">check_for_partition</span><span class="p">(</span><span class="s1">&#39;airflow&#39;</span><span class="p">,</span> <span class="n">t</span><span class="p">,</span> <span class="s2">&quot;ds=&#39;2015-01-01&#39;&quot;</span><span class="p">)</span>
@@ -3377,6 +3506,15 @@ directory, files will be uploaded inside.</li>
 
 <div class="section" id="module-airflow.contrib.hooks">
 <span id="community-contributed-hooks"></span><h3>Community contributed hooks<a class="headerlink" href="#module-airflow.contrib.hooks" title="Permalink to this headline">�</a></h3>
+<p>Importer that dynamically loads a class and module from its parent. This
+allows Airflow to support <cite>from airflow.operators.bash_operator import
+BashOperator</cite> even though BashOperator is actually in
+airflow.operators.bash_operator.</p>
+<p>The importer also takes over for the parent_module by wrapping it. This is
+required to support attribute-based usage:</p>
+<blockquote>
+<div>from airflow import operators
+operators.BashOperator(...)</div></blockquote>
 <dl class="class">
 <dt id="airflow.contrib.hooks.BigQueryHook">
 <em class="property">class </em><code class="descclassname">airflow.contrib.hooks.</code><code class="descname">BigQueryHook</code><span class="sig-paren">(</span><em>bigquery_conn_id='bigquery_default'</em>, <em>delegate_to=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/bigquery_hook.html#BigQueryHook"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.BigQueryHook" title="Permalink to this definition">�</a></dt>
@@ -3500,7 +3638,7 @@ downstream.</p>
 <dt id="airflow.contrib.hooks.FTPHook.close_conn">
 <code class="descname">close_conn</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/ftp_hook.html#FTPHook.close_conn"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.FTPHook.close_conn" title="Permalink to this definition">�</a></dt>
 <dd><p>Closes the connection. An error will occur if the
-connection wasnt ever opened.</p>
+connection wasn&#8217;t ever opened.</p>
 </dd></dl>
 
 <dl class="method">
@@ -3534,7 +3672,7 @@ connection wasnt ever opened.</p>
 <dl class="method">
 <dt id="airflow.contrib.hooks.FTPHook.delete_file">
 <code class="descname">delete_file</code><span class="sig-paren">(</span><em>path</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/ftp_hook.html#FTPHook.delete_file"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.FTPHook.delete_file" title="Permalink to this definition">�</a></dt>
-<dd><p>Removes a file on the FTP Server</p>
+<dd><p>Removes a file on the FTP Server.</p>
 <table class="docutils field-list" frame="void" rules="none">
 <col class="field-name" />
 <col class="field-body" />

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/concepts.html
----------------------------------------------------------------------
diff --git a/concepts.html b/concepts.html
index b41766f..15b2b4d 100644
--- a/concepts.html
+++ b/concepts.html
@@ -573,7 +573,7 @@ object. This will prevent the SubDAG from being treated like a separate DAG in
 the main UI. For example:</p>
 <div class="code python highlight-default"><div class="highlight"><pre><span></span><span class="c1">#dags/subdag.py</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">DAG</span>
-<span class="kn">from</span> <span class="nn">airflow.operators</span> <span class="k">import</span> <span class="n">DummyOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.operators.dummy_operator</span> <span class="k">import</span> <span class="n">DummyOperator</span>
 
 
 <span class="c1"># Dag is returned by a factory method</span>
@@ -596,7 +596,7 @@ the main UI. For example:</p>
 <div class="code python highlight-default"><div class="highlight"><pre><span></span><span class="c1"># main_dag.py</span>
 <span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span>
 <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">DAG</span>
-<span class="kn">from</span> <span class="nn">airflow.operators</span> <span class="k">import</span> <span class="n">SubDagOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.operators.subdag_operator</span> <span class="k">import</span> <span class="n">SubDagOperator</span>
 <span class="kn">from</span> <span class="nn">dags.subdag</span> <span class="k">import</span> <span class="n">sub_dag</span>
 
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/faq.html
----------------------------------------------------------------------
diff --git a/faq.html b/faq.html
index 050da7b..3d8cb5d 100644
--- a/faq.html
+++ b/faq.html
@@ -202,12 +202,15 @@ inactive DagRuns to get the new task to get onboarded properly.</p>
 <code class="docutils literal"><span class="pre">datetime.now()</span></code> as it can be quite confusing. The task is triggered
 once the period closes, and in theory an <code class="docutils literal"><span class="pre">&#64;hourly</span></code> DAG would never get to
 an hour after now as <code class="docutils literal"><span class="pre">now()</span></code> moves along.</p>
-<p>We also recommend using rounded <code class="docutils literal"><span class="pre">start_date</span></code> in relation to your
-<code class="docutils literal"><span class="pre">schedule_interval</span></code>. This means an <code class="docutils literal"><span class="pre">&#64;hourly</span></code> would be at <code class="docutils literal"><span class="pre">00:00</span></code>
+<p>Previously we also recommended using rounded <code class="docutils literal"><span class="pre">start_date</span></code> in relation to your
+<code class="docutils literal"><span class="pre">schedule_interval</span></code>. This meant an <code class="docutils literal"><span class="pre">&#64;hourly</span></code> would be at <code class="docutils literal"><span class="pre">00:00</span></code>
 minutes:seconds, a <code class="docutils literal"><span class="pre">&#64;daily</span></code> job at midnight, a <code class="docutils literal"><span class="pre">&#64;monthly</span></code> job on the
-first of the month. You can use any sensor or a <code class="docutils literal"><span class="pre">TimeDeltaSensor</span></code> to delay
-the execution of tasks within that period. While <code class="docutils literal"><span class="pre">schedule_interval</span></code>
-does allow specifying a <code class="docutils literal"><span class="pre">datetime.timedelta</span></code>
+first of the month. This is no longer required. Airflow will not auto align
+the <code class="docutils literal"><span class="pre">start_date</span></code> and the <code class="docutils literal"><span class="pre">schedule_interval</span></code>, by using the <code class="docutils literal"><span class="pre">start_date</span></code>
+as the moment to start looking.</p>
+<p>You can use any sensor or a <code class="docutils literal"><span class="pre">TimeDeltaSensor</span></code> to delay
+the execution of tasks within the schedule interval.
+While <code class="docutils literal"><span class="pre">schedule_interval</span></code> does allow specifying a <code class="docutils literal"><span class="pre">datetime.timedelta</span></code>
 object, we recommend using the macros or cron expressions instead, as
 it enforces this idea of rounded schedules.</p>
 <p>When using <code class="docutils literal"><span class="pre">depends_on_past=True</span></code> it&#8217;s important to pay special attention

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/genindex.html
----------------------------------------------------------------------
diff --git a/genindex.html b/genindex.html
index ecba6b9..20b510f 100644
--- a/genindex.html
+++ b/genindex.html
@@ -150,6 +150,7 @@
  | <a href="#K"><strong>K</strong></a>
  | <a href="#L"><strong>L</strong></a>
  | <a href="#M"><strong>M</strong></a>
+ | <a href="#N"><strong>N</strong></a>
  | <a href="#O"><strong>O</strong></a>
  | <a href="#P"><strong>P</strong></a>
  | <a href="#Q"><strong>Q</strong></a>
@@ -288,6 +289,10 @@
   </dt>
 
       
+  <dt><a href="code.html#airflow.hooks.HiveMetastoreHook.check_for_named_partition">check_for_named_partition() (airflow.hooks.HiveMetastoreHook method)</a>
+  </dt>
+
+      
   <dt><a href="code.html#airflow.hooks.HiveMetastoreHook.check_for_partition">check_for_partition() (airflow.hooks.HiveMetastoreHook method)</a>
   </dt>
 
@@ -774,19 +779,15 @@
 
       </dl></dd>
       
-  <dt><a href="code.html#airflow.macros.integrate_plugins">integrate_plugins() (in module airflow.macros)</a>
-  </dt>
-
-      
   <dt><a href="code.html#airflow.models.DAG.is_paused">is_paused (airflow.models.DAG attribute)</a>
   </dt>
 
-  </dl></td>
-  <td style="width: 33%" valign="top"><dl>
       
   <dt><a href="code.html#airflow.models.TaskInstance.is_premature">is_premature() (airflow.models.TaskInstance method)</a>
   </dt>
 
+  </dl></td>
+  <td style="width: 33%" valign="top"><dl>
       
   <dt><a href="code.html#airflow.models.TaskInstance.is_queueable">is_queueable() (airflow.models.TaskInstance method)</a>
   </dt>
@@ -914,6 +915,16 @@
   </dl></td>
 </tr></table>
 
+<h2 id="N">N</h2>
+<table style="width: 100%" class="indextable genindextable"><tr>
+  <td style="width: 33%" valign="top"><dl>
+      
+  <dt><a href="code.html#airflow.models.DAG.normalize_schedule">normalize_schedule() (airflow.models.DAG method)</a>
+  </dt>
+
+  </dl></td>
+</tr></table>
+
 <h2 id="O">O</h2>
 <table style="width: 100%" class="indextable genindextable"><tr>
   <td style="width: 33%" valign="top"><dl>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/index.html
----------------------------------------------------------------------
diff --git a/index.html b/index.html
index 383bab0..5ac4744 100644
--- a/index.html
+++ b/index.html
@@ -140,10 +140,10 @@
 <a class="reference internal image-reference" href="_images/incubator.jpg"><img alt="_images/incubator.jpg" src="_images/incubator.jpg" style="width: 150px;" /></a>
 <div class="section" id="apache-airflow-incubating-documentation">
 <h1>Apache Airflow (incubating) Documentation<a class="headerlink" href="#apache-airflow-incubating-documentation" title="Permalink to this headline">�</a></h1>
-<div class="admonition note">
-<p class="first admonition-title">Note</p>
-<p class="last">Apache Airflow is an effort undergoing incubation at The Apache
-Software Foundation (ASF), sponsored by Chris Riccomini.
+<div class="admonition important">
+<p class="first admonition-title">Important</p>
+<p class="last"><strong>Disclaimer</strong>: Apache Airflow is an effort undergoing incubation at The
+Apache Software Foundation (ASF), sponsored by the Apache Incubator.
 Incubation is required of all newly accepted projects until a further
 review indicates that the infrastructure, communications, and
 decision making process have stabilized in a manner consistent with

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/objects.inv
----------------------------------------------------------------------
diff --git a/objects.inv b/objects.inv
index 36f1543..56e566e 100644
Binary files a/objects.inv and b/objects.inv differ