[1/6] incubator-airflow-site git commit: Adding incubator related disclaimer

[ma...@apache.org]

Repository: incubator-airflow-site
Updated Branches:
  refs/heads/asf-site 1bb548101 -> 502934ca8


http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/502934ca/searchindex.js
----------------------------------------------------------------------
diff --git a/searchindex.js b/searchindex.js
index 93399ef..901905a 100644
--- a/searchindex.js
+++ b/searchindex.js
@@ -1 +1 @@
-Search.setIndex({envversion:47,filenames:["cli","code","concepts","configuration","faq","index","installation","license","plugins","profiling","project","scheduler","security","start","tutorial","ui"],objects:{"airflow.contrib":{executors:[1,0,0,"-"],hooks:[1,0,0,"-"],operators:[1,0,0,"-"]},"airflow.contrib.hooks":{CloudantHook:[1,1,1,""],FTPHook:[1,1,1,""],SSHHook:[1,1,1,""],VerticaHook:[1,1,1,""]},"airflow.contrib.hooks.CloudantHook":{db:[1,2,1,""]},"airflow.contrib.hooks.FTPHook":{close_conn:[1,2,1,""],create_directory:[1,2,1,""],delete_directory:[1,2,1,""],delete_file:[1,2,1,""],describe_directory:[1,2,1,""],get_conn:[1,2,1,""],list_directory:[1,2,1,""],retrieve_file:[1,2,1,""],store_file:[1,2,1,""]},"airflow.contrib.hooks.SSHHook":{Popen:[1,2,1,""],check_output:[1,2,1,""],tunnel:[1,2,1,""]},"airflow.contrib.hooks.VerticaHook":{get_conn:[1,2,1,""]},"airflow.contrib.operators":{SSHExecuteOperator:[1,1,1,""],VerticaOperator:[1,1,1,""],VerticaToHiveTransfer:[1,1,1,""]},"airflow.con
 trib.operators.hipchat_operator":{HipChatAPIOperator:[1,1,1,""],HipChatAPISendRoomNotificationOperator:[1,1,1,""]},"airflow.executors":{CeleryExecutor:[1,1,1,""],LocalExecutor:[1,1,1,""],SequentialExecutor:[1,1,1,""]},"airflow.hooks":{DbApiHook:[1,1,1,""],DruidHook:[1,1,1,""],HiveCliHook:[1,1,1,""],HiveMetastoreHook:[1,1,1,""],HiveServer2Hook:[1,1,1,""],HttpHook:[1,1,1,""],MsSqlHook:[1,1,1,""],MySqlHook:[1,1,1,""],PostgresHook:[1,1,1,""],PrestoHook:[1,1,1,""],S3Hook:[1,1,1,""],SqliteHook:[1,1,1,""],WebHDFSHook:[1,1,1,""]},"airflow.hooks.DbApiHook":{bulk_dump:[1,2,1,""],bulk_load:[1,2,1,""],get_conn:[1,2,1,""],get_cursor:[1,2,1,""],get_first:[1,2,1,""],get_pandas_df:[1,2,1,""],get_records:[1,2,1,""],insert_rows:[1,2,1,""],run:[1,2,1,""]},"airflow.hooks.DruidHook":{construct_ingest_query:[1,2,1,""],get_conn:[1,2,1,""],load_from_hdfs:[1,2,1,""]},"airflow.hooks.HiveCliHook":{load_file:[1,2,1,""],run_cli:[1,2,1,""],test_hql:[1,2,1,""]},"airflow.hooks.HiveMetastoreHook":{check_for_partiti
 on:[1,2,1,""],get_databases:[1,2,1,""],get_metastore_client:[1,2,1,""],get_partitions:[1,2,1,""],get_table:[1,2,1,""],get_tables:[1,2,1,""],max_partition:[1,2,1,""],table_exists:[1,2,1,""]},"airflow.hooks.HiveServer2Hook":{get_pandas_df:[1,2,1,""],get_records:[1,2,1,""]},"airflow.hooks.HttpHook":{get_conn:[1,2,1,""],run:[1,2,1,""],run_and_check:[1,2,1,""]},"airflow.hooks.MsSqlHook":{get_conn:[1,2,1,""]},"airflow.hooks.MySqlHook":{bulk_load:[1,2,1,""],get_conn:[1,2,1,""]},"airflow.hooks.PrestoHook":{get_conn:[1,2,1,""],get_first:[1,2,1,""],get_pandas_df:[1,2,1,""],get_records:[1,2,1,""],run:[1,2,1,""]},"airflow.hooks.S3Hook":{check_for_bucket:[1,2,1,""],check_for_key:[1,2,1,""],check_for_prefix:[1,2,1,""],check_for_wildcard_key:[1,2,1,""],get_bucket:[1,2,1,""],get_conn:[1,2,1,""],get_key:[1,2,1,""],get_wildcard_key:[1,2,1,""],list_keys:[1,2,1,""],list_prefixes:[1,2,1,""],load_file:[1,2,1,""],load_string:[1,2,1,""]},"airflow.hooks.SqliteHook":{get_conn:[1,2,1,""]},"airflow.hooks.WebHD
 FSHook":{check_for_path:[1,2,1,""],get_conn:[1,2,1,""],load_file:[1,2,1,""]},"airflow.macros":{ds_add:[1,3,1,""],ds_format:[1,3,1,""],hive:[1,0,0,"-"],integrate_plugins:[1,3,1,""],random:[1,3,1,""]},"airflow.macros.hive":{closest_ds_partition:[1,3,1,""],max_partition:[1,3,1,""]},"airflow.models":{BaseOperator:[1,1,1,""],Connection:[1,1,1,""],DAG:[1,1,1,""],DagBag:[1,1,1,""],TaskInstance:[1,1,1,""]},"airflow.models.BaseOperator":{clear:[1,2,1,""],dag:[1,4,1,""],detect_downstream_cycle:[1,2,1,""],downstream_list:[1,4,1,""],execute:[1,2,1,""],get_direct_relatives:[1,2,1,""],get_flat_relatives:[1,2,1,""],get_task_instances:[1,2,1,""],has_dag:[1,2,1,""],on_kill:[1,2,1,""],post_execute:[1,2,1,""],pre_execute:[1,2,1,""],prepare_template:[1,2,1,""],render_template:[1,2,1,""],render_template_from_field:[1,2,1,""],run:[1,2,1,""],schedule_interval:[1,4,1,""],set_downstream:[1,2,1,""],set_upstream:[1,2,1,""],upstream_list:[1,4,1,""],xcom_pull:[1,2,1,""],xcom_push:[1,2,1,""]},"airflow.models.Con
 nection":{extra_dejson:[1,4,1,""]},"airflow.models.DAG":{add_task:[1,2,1,""],add_tasks:[1,2,1,""],cli:[1,2,1,""],concurrency_reached:[1,4,1,""],crawl_for_tasks:[1,2,1,""],create_dagrun:[1,2,1,""],filepath:[1,4,1,""],folder:[1,4,1,""],get_template_env:[1,2,1,""],is_paused:[1,4,1,""],latest_execution_date:[1,4,1,""],run:[1,2,1,""],set_dependency:[1,2,1,""],sub_dag:[1,2,1,""],subdags:[1,4,1,""],tree_view:[1,2,1,""]},"airflow.models.DagBag":{bag_dag:[1,2,1,""],collect_dags:[1,2,1,""],dagbag_report:[1,2,1,""],get_dag:[1,2,1,""],kill_zombies:[1,2,1,""],process_file:[1,2,1,""],size:[1,2,1,""]},"airflow.models.TaskInstance":{are_dependencies_met:[1,2,1,""],are_dependents_done:[1,2,1,""],clear_xcom_data:[1,2,1,""],command:[1,2,1,""],current_state:[1,2,1,""],error:[1,2,1,""],evaluate_trigger_rule:[1,2,1,""],is_premature:[1,2,1,""],is_queueable:[1,2,1,""],is_runnable:[1,2,1,""],key:[1,4,1,""],pool_full:[1,2,1,""],ready_for_retry:[1,2,1,""],refresh_from_db:[1,2,1,""],run:[1,2,1,""],xcom_pull:[1
 ,2,1,""],xcom_push:[1,2,1,""]},"airflow.operators":{BashOperator:[1,1,1,""],BranchPythonOperator:[1,1,1,""],DummyOperator:[1,1,1,""],EmailOperator:[1,1,1,""],ExternalTaskSensor:[1,1,1,""],GenericTransfer:[1,1,1,""],HdfsSensor:[1,1,1,""],Hive2SambaOperator:[1,1,1,""],HiveOperator:[1,1,1,""],HivePartitionSensor:[1,1,1,""],HiveToDruidTransfer:[1,1,1,""],HiveToMySqlTransfer:[1,1,1,""],HttpSensor:[1,1,1,""],MetastorePartitionSensor:[1,1,1,""],MsSqlOperator:[1,1,1,""],MsSqlToHiveTransfer:[1,1,1,""],MySqlOperator:[1,1,1,""],MySqlToHiveTransfer:[1,1,1,""],PostgresOperator:[1,1,1,""],PrestoCheckOperator:[1,1,1,""],PrestoIntervalCheckOperator:[1,1,1,""],PrestoValueCheckOperator:[1,1,1,""],PythonOperator:[1,1,1,""],S3KeySensor:[1,1,1,""],S3ToHiveTransfer:[1,1,1,""],ShortCircuitOperator:[1,1,1,""],SimpleHttpOperator:[1,1,1,""],SlackAPIOperator:[1,1,1,""],SlackAPIPostOperator:[1,1,1,""],SqlSensor:[1,1,1,""],TimeSensor:[1,1,1,""],TriggerDagRunOperator:[1,1,1,""],WebHdfsSensor:[1,1,1,""]},"airflow
 .operators.BashOperator":{execute:[1,2,1,""]},"airflow.operators.SlackAPIOperator":{construct_api_call_params:[1,2,1,""],execute:[1,2,1,""]},"airflow.operators.docker_operator":{DockerOperator:[1,1,1,""]},"airflow.operators.sensors":{BaseSensorOperator:[1,1,1,""]},airflow:{executors:[1,0,0,"-"],hooks:[1,0,0,"-"],macros:[1,0,0,"-"],models:[1,0,0,"-"],operators:[1,0,0,"-"]}},objnames:{"0":["py","module","Python module"],"1":["py","class","Python class"],"2":["py","method","Python method"],"3":["py","function","Python function"],"4":["py","attribute","Python attribute"]},objtypes:{"0":"py:module","1":"py:class","2":"py:method","3":"py:function","4":"py:attribute"},terms:{"00am":1,"01t23":11,"10gb":1,"10pm":2,"128m":1,"2am":1,"5gb":1,"9ba53fba14eb":[],"__class__":2,"__init__":[1,2],"__name__":[2,8],"_cmd":3,"_host":12,"abstract":[1,2,5,8],"boolean":[1,2],"byte":[1,4],"case":[1,2,4,11,12,14],"class":[0,1,2,6,7,8,14],"default":[],"export":[1,13],"final":1,"float":1,"function":[],"import":
 [],"int":1,"long":1,"new":[1,2,4,8,11,12],"return":[0,1,2,8],"short":1,"static":[1,5,8],"super":[1,12],"switch":[3,12],"true":[1,2,4,12,14],"try":[1,2,3],"var":1,"while":[1,2,4,5,7,11,13,14,15],abbrevi:2,abil:2,abl:[3,4,12,13,14],about:[1,2,3,7,10,11,14],abov:[2,3,7,12,15],absenc:2,absolut:2,accept:[1,7],access:[1,2,3,8,12],access_token:15,accord:12,accordingli:2,account:[1,12],acquir:1,across:[3,15],act:[1,2,7,14],action:[1,15],activ:[1,2,4,8,11,12,14],actual:[1,2,4,14],acycl:[1,2,5],add:[1,2,3,7,12],add_task:1,addendum:7,addit:[],addition:12,addprinc:12,adhoc:[],adjust:[3,12],admin:[2,3,4,8,9,12,13],admin_view:8,advanc:14,advantag:12,advis:[1,7],affect:2,after:[1,2,4,11],afterward:1,against:[1,2,4,7,12,14],agari:10,aggregate_db_message_job:2,aggregate_db_message_job_cmd:2,ago:4,agre:7,agreement:[2,7],airbnb:[1,2,10,14],airflow:[],airflow__:3,airflow__core__sql_alchemy_conn:3,airflow_config:3,airflow_conn_:[2,3],airflow_conn_postgres_mast:2,airflow_hom:[3,4,8,13],airflow_set:2,airf
 low_tmp_dir:1,airflowignor:1,airflowplugin:8,airflowtestplugin:8,aiyzngq:1,alchemi:3,alert:[1,2,8],all:[0,1,2,3,4,6,7,11,12,14],all_db:6,all_don:[1,2],all_fail:[1,2],all_success:[1,2],alleg:7,alloc:1,allow:[0,1,2,3,4,5,8,9,11,12,13,14,15],allowed_st:1,allowed_team:12,alon:7,along:[1,4,7,14],alongsid:[1,7,11],alreadi:1,alright:14,also:[1,2,3,4,6,7,8,11,12,14],alter:[1,2,3,4,12],altern:[1,11],altert:1,although:12,alwai:[1,2,15],amazon:3,amount:[1,2],analys:15,analyz:2,anand:10,anchor:1,ani:[1,2,3,4,7,14,15],annot:7,announc:10,anomali:8,anoth:[1,2,4,10],ansibl:3,anyon:12,anyth:[2,14],anytim:2,anywher:[1,2],aoen:10,apach:[],api:[],api_call_param:1,api_kei:15,api_param:1,api_vers:1,apikei:15,apiv2:1,appar:1,appear:[1,2,7],append:3,appendix:7,appli:[1,2,6,7,11],applic:[7,8,9,12],appropri:[2,7,12],arbitrari:[2,5],architectur:5,archiv:7,are_dependencies_met:1,are_dependents_don:1,aren:[1,2],arg:1,argument:[],aris:7,around:[1,5,14],arrai:[1,5,8],arthur:10,artwr:10,ascii:1,assert:7,asset:1,as
 sign:[],associ:[1,2,7,11],assum:[1,2,3,7,14],asterisk:12,async:6,atom:2,attach:[1,2,7],attach_to:1,attack:12,attempt:[0,1],attent:[1,4],attr:1,attribut:[0,1,2,7],audit:8,auth:[1,12],auth_backend:12,authen:12,authent:[],authmechan:1,author:[1,3,5,7,12,14,15],authorship:7,autocommit:1,automat:[1,2,3],avail:[1,2,3,7,8,9],averag:1,avoid:2,awar:2,azkaban:5,back:[1,4,8],backend:[],backfil:[],background:[1,14],backup:3,bag:1,bag_dag:1,bahchi:[],balanc:2,bar:[1,2],bare:14,base64:4,base:[1,2,3,4,7,8,11],base_executor:[1,8],base_hook:[1,8],base_url:1,basedn:12,baseexecutor:[1,8],basehook:[1,2,8],baseview:8,bash:[1,2,14],bash_1:2,bash_command:[1,2,14],bash_command_to_run:3,bash_queu:14,bashoper:[1,2,14],basi:7,basic:[2,6,8,9,14],bcrypt:12,beauchemin:10,becaus:2,becom:[1,2,5,8,14],beelin:1,been:[1,2,3,7,11,12],befor:[0,1,2,4,12],begin:[2,11],behalf:7,behav:1,behavior:[1,2],behind:[1,3,6,11],belong:12,below:[1,7,8,13,14],benefici:7,best:[1,10],better:14,between:[1,2,14],beyond:[],bin:2,bind:7,bi
 nd_password:12,bind_us:12,bit:3,blob:14,block:[1,2,14,15],blog:[7,10],blueprint:8,bodi:1,boilerpl:7,bolk:10,bolkedebruin:10,bonu:1,bool:1,boot:3,both:[1,4],bother:14,boto:1,bottleneck:15,box:[1,3,13],bracket:[7,14],branch:[],branch_fals:2,branchpythonoper:[1,2],broker:[0,3],broker_api:0,broker_url:3,brought:10,brows:[2,4],bruin:10,bucket:[1,3],bucket_kei:1,bucket_nam:1,buffer:1,bug:3,build:[],built:[1,2,3,5,8,9,14],bulk:[4,15],bulk_dump:1,bulk_load:1,bump:2,bundl:1,burn:0,but_this_dag_will_not:2,c12318391:1,cacert:12,cach:12,call:[1,2,3,14],callabl:1,callback:12,can:[1,2,3,4,5,6,8,9,11,12,13,14,15],cannot:[1,2,7,14],capabl:8,capac:2,card:1,care:[0,3],carri:[2,7],cast:1,cat:1,categori:8,caus:[4,7],celeri:[],celery_result_backend:3,celeryexecutor:[1,2,3,4,6,11],cell:1,central:2,cert:1,certain:[1,2],certif:1,cfg:[1,2,3,4,11,12,13,14],chain:2,chang:[3,5,7,11,12,13,14],channel:[1,10],charact:7,character:2,charg:7,charset:1,chart:[],chat:10,check:[0,1,2,4],check_for_bucket:1,check_for_kei
 :1,check_for_partit:1,check_for_path:1,check_for_prefix:1,check_for_wildcard_kei:1,check_head:1,check_oper:1,check_output:1,checkoper:1,chef:[3,10],child:[1,2],child_dag:2,child_dag_nam:2,chmod:12,choic:[0,1,14],choos:[1,2,7],chri:10,circuit:1,claim:7,clariti:5,claus:1,clean:1,cleanup:1,clear:[0,1,2,4,11,15],clear_xcom_data:1,cli:[1,2,3,4,11],click:[4,12,14,15],client:[1,6,12],client_id:12,client_secret:12,close:[1,4,12],close_conn:1,closest:1,closest_ds_partit:1,cloud:[3,6],cloudant:[1,6],cloudant_conn_id:1,cloudant_default:1,cloudanthook:1,cmd:1,code:[],col:1,collabor:5,collect:[1,2,8],collect_dag:1,color:1,column:1,com:[1,12,14],combin:[2,7],come:[1,11],comma:[0,1,2],comment:7,commerci:7,commit:[1,10,12],commit_everi:1,common:[2,3,4,7,10,14],commonli:1,compani:8,compar:5,compil:[4,7],complet:[1,2,3],complex:[1,2,5,8],compli:7,complianc:7,complic:14,compon:8,compos:[2,12,14],comprehens:15,comput:7,conceptu:2,concern:2,concur:4,concurr:[0,1,4],concurrency_reach:1,condit:[1,2,6,7],c
 onf:[0,1,3],config:[8,12],configur:[],confirm:[0,4,11],confluenc:[],confus:[1,2,4],conjunct:[0,2,13],conn_id:[1,2,3],conn_typ:1,connect:[],connect_timeout:1,connecton:9,consequenti:7,consid:[1,2,3],consider:1,consist:[2,7],conspicu:7,constant:[1,2],constitu:2,constitut:7,constraint:1,constru:7,construct:1,construct_api_call_param:1,construct_ingest_queri:1,constructor:[1,14],contact:[3,7],contain:[1,2,3,7,11,12,14,15],content:[],context:[],continu:[1,5],contract:7,contrib:[1,2,12],contributor:[7,10],contributori:7,control:[1,2,7,15],conveni:[1,2],convent:2,convers:7,cookbook:[],cool:1,coordin:1,copi:[1,3,7,12],copyright:[7,12],core:[],correct:2,correpsond:2,correspond:[1,2,3,4],could:[1,2,14],count:1,counterclaim:7,cours:12,cover:11,cpu:[1,8],craft:1,crawl:[1,11],crawl_for_task:1,creat:[1,2,3,4,8,9,11,12,13,14,15],create_dagrun:1,create_directori:1,creation:1,credenti:12,credit:12,criccomini:10,criteria:[1,2],criterion:1,critic:[1,2],cron:[1,4,11],cross:[2,7,14],crt:12,crypto:[3,4,6
 ],csv:1,curli:14,current:[1,15],current_st:1,cursor:1,custom:8,customari:7,cwiki:[],cycl:[1,2,14],daemon:[0,1,3],dag:[],dag_1:2,dag_2:2,dag_fold:[1,2],dag_id:[0,1,2,14],dag_md:2,dag_run:1,dag_run_obj:1,dagbag:1,dagbag_report:1,dagfil:2,dagrun:[0,1,4],dagrun_timeout:1,dags_fold:3,dai:[1,11,13,14],daili:[1,4,11],damag:7,dan:[1,10],data:[],data_profiler_filt:12,databas:[0,1,2,3,5,6,8,9,13,14],datafram:1,dataset:1,datasourc:1,date:[0,1,2,7,11,14],date_filter_column:1,datestamp:1,datetim:[1,2,4,11,14],dateutil:1,davydov:10,days_back:1,dbapi_hook:1,dbapihook:1,deadlock:[],deal:4,debug:[0,1,14],decid:[1,3],declar:1,deep:1,def:[1,2,8],default_arg:[1,2,4,14],default_login:12,default_queu:2,defend:7,defer:2,deferred_op:2,defin:[1,2,3,4,5,7,8,11,12,14],definit:[],del:1,delai:[1,4],delegate_to:[],delet:[1,2,11,15],delete_directori:1,delete_fil:1,deliber:[1,7],delimit:[0,1,2],demonstr:2,depend:[],depends_on_past:[0,1,2,4,14],deploi:8,deploy:8,depth:14,deriv:[1,3,7,8],descend:2,describ:[1,2,7],de
 scribe_directori:1,descript:[1,2,7],deseri:[0,1,2],deserialize_json:2,design:[2,7,11],desir:[1,14],destin:1,destination_conn_id:1,destination_filepath:1,destination_t:1,detail:[1,2,4,12,15],detect:[1,8],detect_downstream_cycl:1,determin:[1,2,7],dev:[3,6,10],devel:6,devel_hadoop:6,develop:[0,1,12,14],deviat:1,dgaoqozlvea:[],dialog:11,dict:[0,1,2],dictionari:[1,2,14],did:14,die:2,differ:[1,2,7,8,14,15],dimens:1,direct:[1,2,3,5,7],directli:[1,2,4,9],directori:[0,1,2,3,12,14],dirti:13,disabl:1,disclaim:7,discuss:7,disk:1,displai:[3,7,11,14],disregard:[1,14],distinct:1,distribut:[1,2,6,7],do_pickl:0,doc:[1,2,4,14],doc_json:2,doc_md:2,doc_rst:2,doc_yaml:2,docker:[1,10],docker_oper:1,docker_url:1,dockeroper:[1,2],document:[],doe:[0,1,2,3,4,6,7,12,14],does_not_exist:1,doesn:[1,2,14],domain:12,don:[1,2,3,6,7,11],done:[1,2,11,12,14],donot_pickl:[0,1],door:2,dot:[1,2],doubl:[1,3,14],down:[0,2],download:[1,3],downstream:[0,1,2,11],downstream_list:1,downstream_task_id:1,drive:3,driven:8,driver:3
 ,drop:[1,8],druid:[1,6],druid_datasourc:1,druid_ingest_conn_id:1,druid_ingest_default:1,druid_query_conn_id:1,druid_query_default:1,druidhook:1,dry:0,dry_run:0,ds_add:[1,14],ds_format:1,ds_nodash:1,dubiou:1,dummi:[1,2,8],dummy_1:2,dummy_oper:2,dummy_task:2,dummyoper:[1,2],dump:1,durat:[],dure:[3,15],dynam:[1,2,4,5],each:[1,2,4,7,10,11,14],earlier:14,easi:[2,3,5,8,9,12,15],easier:[1,7],easiest:6,easili:[1,2,5,9,14],echo:[2,14],ecosystem:8,edit:[3,15],editori:7,effect:2,effici:1,egg:3,either:[1,2,3,7,12,13],elabor:7,elaps:1,electron:7,eleg:5,element:3,els:13,elsewher:2,email:[1,2,10,12,14],email_on_failur:[1,14],email_on_retri:[1,14],emailoper:[1,2],emit:[2,14],empti:[1,2],enabl:[],enclos:7,encod:[1,4],encrypt:[1,3,4,6,12],encrypt_s3_log:3,end:[1,7,11],end_dat:[0,1,14],endpoint:1,enforc:[2,4],engin:[1,4,5],enterpris:[],entir:[0,4],entiti:7,entri:[1,11],env:[1,2],environ:[1,2,3,5,6,9,11,14,15],ep_data_pipeline_db_msg_agg:2,equival:[2,6,14],era:4,error:1,especi:[2,4,14],essenti:1,etc:[1
 ,2,3,12],eval:1,evalu:[1,2,4,11,14],evaluate_trigger_rul:1,even:[1,2,7,9],event:[2,7,14],eventlet:0,ever:1,everi:[1,2,11,12,14],everyon:14,everyth:[14,15],exactli:[1,15],exampl:[],example1:13,example_bash_oper:13,example_dag:[2,14],example_team_1:12,example_team_2:12,exce:1,except:[1,2,4,7,14],exchang:[2,5],exclud:7,exclus:[7,11],execut:[0,1,2,3,4,5,7,11,14],execution_d:[0,1,2,14],execution_delta:1,execution_timeout:[1,2],exercis:7,exhaust:3,exist:[0,1,2,3,4,14],exit:[0,12],expect:[1,2,4,5],expir:1,explain:1,explan:14,explicit:5,explicit_op:2,explicitli:[1,2,7,12,14],expos:[1,2,3,8,12],express:[1,4,7,11],ext:1,extend:5,extens:[1,5],extern:[],external_dag_id:1,external_task_id:1,external_trigg:1,externaltasksensor:1,extra:[],extra_dejson:1,extra_opt:1,fact:2,factori:2,fail:[0,1,2,3,4,11,14,15],failur:[1,3,4,7],fairli:13,fall:10,fals:[0,1,3,11,14],familiar:[2,14],faq:[],fast:13,fault:2,featur:[1,2,3,6,8,14,15],feb:12,fee:7,fernet:4,fernet_kei:4,fetch:3,few:[1,3,9,13,14],field:[1,7],fi
 eld_dict:1,fifti:7,figur:[1,2,11],file:[],filenam:[1,2],filepath:1,filestatu:1,filesystem:3,fill:[1,2,9,12],filter:[0,1,2,12],filter_by_own:12,find:[1,3,4,14,15],fine:14,finish:1,fire:[1,2,3],first:[0,1,2,3,4,10,11,13,14],fit:[1,5,7],fix:[1,11],flag:[1,11],flag_upstream_fail:1,flask:[0,8,9],flask_admin:8,flask_blueprint:8,flask_login:12,flat:1,flavor:8,flexibl:[1,14],flower:[0,3],folder:[1,3,8,11,13,14],follow:[1,2,3,5,7,12,14],foo:[1,2],foo_dag_id:4,foobar:3,forc:[0,1],force_pul:1,foreground:0,foreign:1,form:[],format:[1,2,3,7],forward:[1,4],found:[1,3,12],foundat:[7,10,13],four:2,framework:[3,8],free:[2,7],frm:1,from:[0,1,2,3,4,5,7,8,9,10,11,12,13,14,15],ftp:1,ftp_conn_id:1,ftp_default:1,ftphook:1,full:[1,10],full_filepath:1,fulli:[4,12],functiontyp:1,fundament:14,further:[13,14],futur:[1,11],galaxi:8,gantt:[],garag:2,gate:12,gc_base_hook:[],gcloud:[],gcp_api:6,gender:1,gener:[0,1,2,5,7,8,12,14,15],generictransf:1,get:[],get_active_run:[],get_bucket:1,get_conn:1,get_connect:2,get_
 cursor:1,get_dag:1,get_databas:1,get_direct_rel:1,get_first:1,get_flat_rel:1,get_kei:1,get_metastore_cli:1,get_pandas_df:1,get_partit:1,get_partitions_by_filt:1,get_record:1,get_tabl:1,get_task_inst:1,get_template_context:1,get_template_env:1,get_wildcard_kei:1,gevent:0,ghe:[],ghe_oauth:12,ghost:1,git:3,github:[],github_enterpris:12,github_enterprise_auth:12,githubusercont:1,gitignor:1,gitter:10,give:[2,7],given:[1,2],glanc:15,global:[1,2,4],goal:14,goe:[1,4,14],good:2,goodby:2,goodwil:7,googl:[3,6],google_cloud_storage_conn_id:[],google_cloud_storage_default:[],googlecloudbasehook:[],googlecloudstoragehook:[],govern:7,grab:1,grade:3,grai:1,grant:7,graph:[],great:3,greater:1,green:1,grossli:7,group:[1,3,12],gunicorn:[0,6],hack:1,had:[1,4],hadoop:[6,10,12],hadoop_dependency_coordin:1,hand:[12,13],handi:10,handl:[1,2,3],happen:[2,11,14],hard:[1,2],harmless:7,has_dag:1,hash:[1,12],hasn:2,have:[1,2,3,4,6,7,8,9,11,12,14],haven:[1,2,14],hdf:[1,2,6],hdfs_conn_id:1,hdfs_default:1,hdfs_path:
 1,hdfscli:1,hdfssensor:1,head:[13,14],header:1,heartbeat:[1,2],heavier:1,hello:[2,8],help:[0,7,8,12],here:[1,2,3,4,6,8,10,11,13,14,15],herebi:7,herein:7,hidden:15,hierarchi:[1,14],high:1,highchart:9,higher:1,highli:3,highlight:14,hipchat:1,hipchat_oper:1,hipchatapioper:1,hipchatapisendroomnotificationoper:1,hit:[1,2],hive2mysqloper:1,hive2sambaoper:[1,6],hive:[1,2,3,6,8,12],hive_cli_conn_id:1,hive_cli_default:1,hive_cli_param:1,hive_conn_id:1,hive_t:1,hiveclihook:1,hiveconf:1,hiveconf_jinja_transl:1,hivemetastorehook:1,hiveoper:[1,2,3,12],hivepartitionsensor:[1,2],hivesensoroper:1,hiveserver2:1,hiveserver2_conn_id:1,hiveserver2_default:1,hiveserver2hook:1,hivetodruidtransf:1,hivetomysqltransf:1,hold:7,home:13,homogen:3,hope:11,horribl:14,host:[1,3,12],hostnam:[0,1,2,3],hour:[1,2,4,11],hourli:[1,4,11],hous:2,how:[1,2,3,4,7,8,11,14,15],howev:[2,7,12],hql:1,html:[1,8,14],html_content:1,http:[1,2,7,8,12,14],http_conn_id:1,http_default:1,httphook:1,httpoper:2,httpsensor:1,human:1,icon:1,
 icon_url:1,idea:[],idempot:1,identif:7,identifi:[1,7,14,15],ignor:[0,1],ignore_depend:[0,1],ignore_depends_on_past:[0,1],ignore_first_depends_on_past:[0,1],iii:7,imag:1,immedi:[1,2],imper:3,imperson:12,implement:[1,2],impli:[1,7],improv:7,impyla:1,inabl:7,inact:4,inc:7,incident:7,includ:[0,1,2,3,7,11,14],include_adhoc:[0,1],include_downstream:1,include_exampl:1,include_prior_d:1,include_queu:1,include_upstream:1,inclus:7,incorpor:7,incub:[],incur:7,indemn:7,indemnifi:7,indentifi:0,independ:[1,2],index:1,indic:[1,2,7],indirect:7,individu:[1,7,11,14],infer:[1,2],inferred_op:2,infil:1,infin:5,info:1,inform:[1,2,3,7,11,12,14],infrastructur:1,infring:7,ingest:1,inherit:[1,14],init:3,initctl:3,initdb:[0,3,13],initi:[0,3,12,13],initialis:1,inject:8,input:1,input_format:1,insecur:12,insecurecli:1,insert:[1,2],insert_row:1,insid:[1,2],inspect:[11,13],instal:[],instanc:[],instanti:[],instati:14,instead:[0,1,4],insteadnhttp:1,institut:7,instruct:2,insur:1,integr:[],integrate_plugin:1,intention
 :7,inter:2,interact:[1,2,3,8,9],interest:[1,14],intergr:8,intern:0,interpret:[1,2],interv:1,intervalcheckoper:1,introduc:4,intuit:14,invari:[1,2],invok:1,ipython:2,irrevoc:7,is_paus:1,is_prematur:1,is_queueabl:1,is_runn:1,ish:1,isn:[1,2,4],isoformat:1,issu:[1,2,5,7,10],item:2,iter:1,itself:[2,12,14],januari:[7,11],java:1,jdbc:[1,6],jeremiah:10,jinja2:1,jinja:[],jinja_env:1,jira:[],jlowin:10,job:[0,1,2,3,4,11,13,15],job_id:1,jobtrack:1,join:[2,10],json:[0,1,2],jupyt:2,just:[0,1,2,4,8,14],kadmin:12,kapil:3,keep:[],kei:[0,1,2,3,4,12,15],kerber:[6,12],kerbero:[],key_fil:1,key_path:[],keytab:[0,12],keyword:1,kick:[3,11],kill:[1,2],kill_zombi:1,kind:7,know:[3,14],knowledg:8,known:6,known_host:1,kruyk:10,kwarg:1,kwd:1,lab:1,label:[1,9],lai:13,lambda:[1,2],land:[1,8],languag:[7,14],larg:1,larger:1,last:1,late:15,later:9,latest:[0,1,6,14],latest_d:1,latest_execution_d:1,launch:12,law:7,lawsuit:7,layout:14,ldap3:12,ldap:[],ldap_auth:12,ldap_ca:12,lead:[1,2],lean:5,least:[1,2,7],leav:1,led:2,l
 eft:[2,12],legaci:4,legal:7,len:1,leo:10,less:1,lesson:10,let:[0,1,2,3,9,14,15],level:[1,2,5,11,12],leverag:[1,2,14],liabil:7,liabl:7,lib:[1,2,3],libffi:3,librari:[1,2,3,5,8,14],libz:2,licens:[],licensor:7,life:2,light:[1,2],lighter:1,lightweight:2,like:[1,2,3,6,8,12,14],limit:[],link:[],linkedin:[],list:[0,1,2,3,4,6,7,8,10,12,14,15],list_dag:[0,4,14],list_directori:1,list_kei:1,list_prefix:1,list_task:[0,4,14],listen:2,lit:1,liter:1,litig:7,live:1,load:[1,2],load_fil:1,load_from_hdf:1,load_str:1,local:[0,1,2,3,12,14],local_executor:1,local_full_path_or_buff:1,local_infil:1,local_port:1,localexecutor:[0,1,2,3,11],localhost:2,locat:[0,1,2,3,12,14],lock:1,lock_for_upd:1,log:[],log_fil:0,loggingmixin:1,logic:[1,2,14],login:[1,2,3,12],logist:8,look:[0,1,2,5,8,10,11,14],loss:7,lot:11,lowin:10,machin:[2,3,12],made:[1,2,7],mai:[1,2,3,4,7,11,14],mail:[7,10],maill:[],main:[1,2,4,8],main_dag:2,maintain:[1,5,10],make:[1,2,3,4,5,7,12,14,15],malfunct:7,malik:3,man:6,manag:[],mani:[0,1,2,4,8,14,1
 5],manual:[1,2,3,11],map:1,mapr:1,mark:[0,1,2,7,11],mark_success:[0,1,11],markdown:2,markup:1,master:[1,2,14],match:[0,1,2],matter:1,max:[1,10],max_active_run:[1,4],max_partit:1,maxim:10,maxime_beauchemin:0,maximum:1,mayb:2,mean:[1,2,3,4,7,12,14],meaning:1,meant:1,mechan:[1,7,12],media:7,medium:7,meet:[1,7],mem_limit:1,member:12,memberof:12,memori:1,menu:[],menu_link:8,menulink:8,merchant:7,mere:7,merg:14,mesosexecutor:[3,11],messag:[1,2,5],message_format:1,met:[1,3,4,11],metadata:[],metastor:1,metastore_conn_id:1,metastore_default:1,metastore_mysql:1,metastorepartitionsensor:1,method:[1,2],metric:[1,8],metric_spec:1,metrics_threshold:1,microsoft:[1,6],midnight:[4,11],might:[1,2,4,11,14],migrat:12,mime:[],mime_typ:[],min:4,mind:[],minimum:[1,6,11],minut:[2,4,11,14],mismatch:2,miss:[1,2],mistercrunch:10,mix:14,mkdir:2,mlsd:1,mode:[0,1,14],modif:7,modifi:[1,2,3,7,9],modul:[],modular:5,monitor:[2,3,5,8,11,15],monospac:2,month:[4,11],monthli:[4,11],more:[1,2,3,5,7,8,11,12,14,15],morn:11
 ,most:[1,2,12,14,15],mostli:[1,5,11],mount:[1,3],move:[1,2,4,5],msg:1,mssql:[1,6],mssql_conn_id:1,mssql_default:1,mssqlhook:1,mssqloper:1,mssqltohivetransf:1,much:[1,2,11],multi:[],multipart:1,multipart_byt:1,multipl:[1,2],multipli:1,multiprocess:1,must:[1,2,3,4,7,12,14],mutat:2,my_conn_str:3,my_dag1:2,my_dag2:2,my_dag:2,my_databas:1,my_funct:2,my_param:14,my_tabl:1,my_useful_packag:2,myarg:1,mypackag:12,mys3conn:3,mysql:[1,2,3,6],mysql_conn_id:1,mysql_default:1,mysql_postoper:1,mysql_preoper:1,mysql_tabl:1,mysqldb:1,mysqlhook:1,mysqloper:[1,3],mysqltohivetransf:1,name:[1,2,3,7,8,12],namespac:[1,8],nav:12,navig:12,ndfor:14,necessarili:7,need:[1,2,3,4,5,6,8,11,12,13,14],neg:[1,11],neglig:7,neighbour:1,nerd:[],nest:[1,14],network:[1,12],network_mod:1,never:[0,1,4],new_user_email:12,new_user_nam:12,newobject:1,next:[],nhere:1,nice:2,night:2,nlst:1,no_confirm:0,no_host_key_check:1,node:[0,1,2],non:[1,3,7],none:[1,2,8,11],noram:12,norandkei:12,normal:[1,2,7],notat:1,note:[],notebook:2,no
 th:[1,7],notic:[2,7,14],notif:1,notifi:1,notwithstand:7,nov:10,now:[1,2,3,4,12,14],nuanc:2,num:1,num_run:0,num_shard:1,number:[0,1,2,3,5],o2snqeoz7:[],oauth2:12,oauth_callback_rout:12,oauth_key_from_github_enterpris:12,oauth_secret_from_github_enterpris:12,obj:1,object:[0,1,2,3,4,7,8,11,14],objectclass:12,oblig:7,observ:1,obtain:7,occur:[1,2],occurr:1,octet:[],off:[1,3,11],offer:[7,8],offici:[1,10],offset:1,often:[1,2],on_failure_callback:1,on_kil:1,on_retry_callback:1,on_success_callback:1,onboard:4,onc:[1,2,3,4,11,14],one_fail:[1,2],one_success:[1,2],onli:[0,1,2,3,4,6,7,11,12,13],only_fail:0,only_if_upd:1,only_run:0,oozi:5,op1:2,op2:2,op3:2,op4:2,op_arg:1,op_kwarg:1,open:[1,2,4,10,12],oper:[],oppos:[1,2],optim:1,option:[],orchestr:[1,5],order:[1,2,3,11,12],org:[7,8,10,14],organ:[2,8],organiz:12,origin:7,orm:1,other:[1,2,3,5,7,9,11,14],otherwis:[1,7,14],our:14,out:[],outag:8,outgrow:13,outlier:15,output:[1,14],output_encod:1,output_format:1,outsid:11,outstand:7,over:[0,1,2,3,13,15]
 ,overlap:15,overrid:[0,1,14],overridden:[1,4],overview:15,overwhelm:2,overwrit:1,own:[],owner:[1,2,7,12,14],ownership:7,oyp49mbwh60:[],p12:[],packag:[],package1:2,page:[2,4,7,10,15],pai:[1,4],pair:[1,15],panda:1,parallel:[1,2,3,13],param:[0,1,14],paramet:[0,1,2,3,4,9,14],parameter:[2,5,9],parent:2,parent_dag:[1,2],parent_dag_nam:2,pars:[1,4,8,14],part:[1,2,7,9,12],parti:7,particular:7,partit:[1,2],partition_nam:1,partli:4,pass:[1,2,4,8,14],pass_valu:1,passwd:15,password:[],password_auth:12,passwordus:12,past:[1,2,4,11,15],patent:7,path2:1,path:[1,2,3],patrick:10,patrickleotardif:10,pattern:[1,2],paus:[0,1],payload:1,pem:1,peopl:[1,8,14],percent:7,perfect:2,perform:[0,1,2,3,5,7,15],perhap:[2,15],period:[1,2,4,11,14],permiss:[7,12],perpetu:7,persist:[1,11],perspect:2,pertain:7,phase:8,pick:[2,3,9],pickl:[0,1,2,3],picklabl:1,pickle_id:1,pickleabl:1,pid:[0,13],pig:[1,2],piggi:8,pigoper:1,pin_100:1,pip:[2,6,13],pipelin:[],pitfal:10,place:[1,2,7,14],placehold:1,plain:[1,3],plan:4,platform
 :[2,5,6],pleas:[2,3,10,12,14],plug:8,plugin:[],pluginexecutor:8,pluginhook:8,pluginoper:8,plugins_manag:8,png:1,point:[1,2,3,4,13,14],poke:1,poke_interv:1,polymorph:1,pool:[],pool_ful:1,popen:1,popul:14,port:[0,1,3,12,13],posit:[0,1],possibl:[0,1,2,3,7,12,13,14,15],post:[1,10],post_execut:1,postgr:[1,2,3,6],postgres_conn_id:1,postgres_default:1,postgres_mast:2,postgreshook:1,postgresoper:1,potenti:2,power:[2,5,7,14],practic:[1,10],pre:4,pre_execut:1,preced:[1,3,14],prefer:[7,11,13],prefix:[1,2,3],preoper:1,prepar:[2,7],prepare_templ:1,prepped_request:1,present:1,preset:11,presto:1,presto_conn_id:1,presto_default:1,prestocheckoper:1,prestohook:1,prestointervalcheckoper:1,prestotomysqloper:2,prestovaluecheckoper:1,pretti:14,prevent:[1,2,12],previou:[0,1,2,4,14],previous:2,primari:1,primit:1,princip:[0,2,12],print:[1,2,7,14],print_dat:14,prior:1,priorit:2,prioriti:[1,2],priority_weight:[1,2,14],privaci:7,probabl:[3,4,12],problem:2,problemat:2,proce:[1,4],process:[0,1,2,3,11,14],process
 _fil:1,product:[1,2,3,5,7,9,11,14],profil:[],program:10,programmat:5,progress:[1,5,14],project:[],promin:7,prompt:0,propag:[1,2],propaget:1,proper:14,properli:[3,4],properti:[1,2,4,12],provid:[1,2,3,7,9,14,15],provide_context:[1,2],proxy_us:[1,12],proxyus:12,psycopg2:3,ptarget:1,publicli:7,publish:1,pull:[1,2],pull_funct:2,puls:[],puppet:[3,10],pure:2,purpl:1,purpos:[7,14],push:[1,2,8],push_funct:2,pushing_task:2,put:[1,2,8],pwd:2,pyhiv:1,pypi:[6,13],python:[1,2,3,5,6,8,12,14],python_1:2,python_cal:[1,2],python_oper:1,pythonhost:8,pythonoper:[1,2],pythonpath:[3,12],qualif:12,qualifi:12,qualiti:[1,2],qubol:6,queri:[],queu:[1,2,4],queue:[],quick:[],quickli:[9,13,14,15],quit:4,r39132:10,rabbitmq:[3,6],rais:[1,14],ran:0,randkei:12,random:1,randomli:2,rang:[0,1,11,14],rather:2,ratio:1,raw:1,reach:[1,2,3,4],reactiv:4,read:[1,3,4,12,14],readabl:[1,7,12],readi:[1,5,13],readm:3,readthedoc:14,ready_for_retri:1,real:3,realli:[1,12,14],realm:12,reason:[4,7,14],reboot:2,rebuild:0,recap:[],receiv
 :[1,2,7,11],recent:[1,2],recip:10,recipi:7,recogn:1,recommend:[1,3,4,7],record:[0,1,2,14],recreat:1,recurs:1,red:1,redhat:3,redi:3,redirect:0,redistribut:7,redund:14,referenc:[2,14],reflect:[2,8,14],refrain:2,refresh:1,refresh_from_db:1,regard:[1,7],regardless:[0,1],regex:[0,1],regex_kei:1,regexp:0,regist:[8,9,12],regular:[1,11],reinit_frequ:12,rel:[1,11,14],relat:[1,2,3,4,6],relationship:2,relativedelta:1,relev:4,reli:[1,12],reliabl:1,remain:[7,12],rememb:[1,14],remot:[1,3,11],remote_base_log_fold:3,remote_full_path:1,remote_host:1,remote_log_conn_id:3,remote_port:1,remov:1,render:[0,1,2,8,14],render_templ:1,render_template_from_field:1,renew:[0,12],repeat:2,repl:12,replac:[1,7,12],report:1,repositori:[3,14],repres:[1,2,4,7,11],represent:[1,15],reproduc:7,reproduct:7,request:[0,1,2],requir:[1,2,3,6,7,12],resembl:2,reset:0,resetdb:0,resid:2,resolv:1,resourc:[],respect:[0,1,2,14],respons:[1,7],response_check:1,rest:1,restart:[2,3,4],restrict:12,restructuredtext:2,result:[1,2,7,9,14],
 retain:7,retri:[1,2,14],retriev:[1,2],retrieve_fil:1,retry_delai:[1,14],return_valu:1,reus:8,revis:7,riccomini:10,rich:[0,2,5,15],rid:1,right:[1,2,7,9],rightmost:2,risk:7,role:3,roll:[],room:1,room_id:1,root:[1,2],root_dag:1,round:[1,4],rout:12,routin:[1,2],row:[1,2],royalti:7,rule:[],run:[],run_a:1,run_and_check:1,run_as_own:[1,12],run_cli:1,run_id:[0,1,11],runme_0:13,runnabl:[1,2],rwc:12,s3_conn_id:1,s3_default:1,s3_kei:1,s3connect:1,s3filetransferoper:2,s3hook:[1,3],s3keysensor:[1,6],s3prefixsensor:6,s3tohivetransf:1,safe_mod:1,sai:[1,2,4,11],said:2,samaccountnam:12,samba:[1,6],samba_conn_id:1,samba_default:1,same:[1,2,3,7,9,12],sampl:3,sandbox:3,save:[3,4,9,14],scalabl:5,scan:[1,2],scene:[6,11],schedul:[],schedule_interv:[1,2,4,11,14],scheduler_interv:4,scheduler_run:3,schema:[1,2,3],scratch:14,screen:14,screenshot:[],script:[],script_begin_tag:1,search:[1,2],search_path:3,search_scop:12,second:[1,2,4,14],secret:[12,15],section:[1,2,3,4,7,12,13,14],secur:[],see:[1,2,3,4,7,11,12,
 13,14,15],seen:[3,15],segment:1,select:[1,2,9,11,12],self:[1,8],sell:7,semicolon:1,send:[0,1,2,10],send_room_notif:1,sender:1,sensor:[1,2,4],sensor_queu:2,sent:[1,2,3,7],separ:[2,7,14],sequenti:[1,11,13],sequentialexecutor:[1,2,13],serial:[0,1],serv:[0,3,14],serve_log:[0,3],server:[0,1,3,8,12,13,14],servic:[0,1,2,3,6,7,11,12],service_account:[],session:[1,12],set:[],set_depend:[1,14],set_downstream:[1,2,14],set_the_password:12,set_upstream:[1,2,14],setgid:3,setuid:3,setup:[3,12],sever:2,shall:7,shape:11,share:[1,2,3,7,8,9],shell:12,ship:[0,1],ship_dag:0,shop:7,shortcircuitoper:1,shortcut:[3,15],shortest:1,should:[1,2,3,4,7,8,11,13,14],shouldn:[1,2],show:[0,1,2,4,8,15],shown:1,sid:10,siddharth:10,side:[1,3],signal:1,similar:[2,3,5],simpl:[1,2,8,9,14],simplehttpoper:1,simpler:14,simplest:12,simpli:[1,2,8,11,12,14],simplifi:1,simul:14,simultan:2,sinc:[1,3,13,14],sing:1,singl:[0,1,2,14],site:12,size:[1,2],skew:8,skip:[0,1,2],sla:[],sla_miss_callback:1,slack:[1,6],slack_oper:1,slackapiop
 er:1,slackapipostoper:[1,6],slackoper:2,slave:3,sleep:14,slightli:5,slot:[1,2],slowli:5,slug:12,small:[1,2],smaller:1,smallish:1,snap:5,sock:1,soft_fail:1,softwar:[7,10],sole:7,solut:5,some:[1,2,3,4,11,14,15],somehow:3,someth:[2,12,14],sometim:[1,2,14],somewhat:14,somewher:13,soon:[1,2,3,4,11],sort:2,sound:[2,14],sourc:[1,2,7,9,10,15],source_conn_id:1,space:5,span:15,spark:[2,5],spawn:3,special:[1,2,4,7],specif:[0,1,2,3,4,7,11,14,15],specifi:[0,1,2,3,4,5,11,12,14],specific_valu:1,spent:15,spring:10,sql:[1,2,3,6,9],sql_alchemy_conn:3,sql_alchemy_conn_cmd:3,sqlachemi:1,sqlalchemi:[1,2,3],sqlite:[1,13],sqlitehook:1,sqloper:2,sqlsensor:1,squeez:1,sscursor:1,ssh:[1,12],ssh_default:1,ssh_hook:1,sshexecuteoper:1,sshhook:1,sshpass:1,ssl:1,sslcert:1,sslmode:1,stabl:6,stack:[6,8],stage:[1,2],stai:11,stamp:11,stand:2,standalon:2,standard:[1,2],stanza:3,start:[],start_dat:[0,1,2,4,11,14],stat:1,state:[0,1,2,4,7,11,14],state_d:1,statement:[1,2,7],static_babynam:1,static_babynames_partit:1,static
 _fold:8,static_path:1,static_url_path:8,statu:[0,1,2,3,4,7,11,13,14,15],stderr:0,stdout:[0,1,14],step:[1,3,14,15],steven:10,still:[1,4],sting:1,stop:[1,3],stoppag:7,storag:3,store:[1,2,3,8,12,13],store_fil:1,storm:5,str:[1,8,11],straightforward:13,stream:[1,5],string:[0,1,2,3,14],string_data:1,structur:[1,5,14],style:1,sub:[0,1],sub_dag:[1,2],sub_part:1,subcommand:[1,3,11,14],subdag:[],subdagoper:2,subdir:0,subdirectori:2,subject:[1,7],sublicens:7,submiss:[7,11],submit:7,subpackag:[6,12],subpartit:1,subprocess:[1,11],subscrib:[2,10],subsect:[0,11],subsequ:[0,7,11],subset:1,subtl:2,subtract:1,subtre:12,succe:[1,2],succeed:[0,1,2,4,15],success:[0,1,2,4,11,14],successfulli:[1,2,14],suit:5,sum:2,summit:10,sundai:11,super_us:12,supersed:7,superus:12,superuser_filt:12,suppli:[3,12],support:[0,1,3,6,7,12],sure:[1,2,3,4,12,14],surfac:14,surgeri:5,sync:[0,1,3,11],sync_to_db:1,synchron:[1,3],syntax:7,sysconfig:3,system:[1,2,3,7],syvineckruyk:10,tab:[1,12],tabl:[1,2,8],table_exist:1,table_nam:
 [1,2],tablenam:1,tag:4,take:[1,2,3,10,12,13,14,15],talk:[1,10,14],tardif:10,target:[1,3,11],target_field:1,target_partition_s:1,target_tim:1,task:[],task_id:[0,1,2,11,14],task_inst:[1,2],task_instance_key_str:1,task_or_task_list:1,task_param:0,task_regex:[0,1],task_stat:0,taskinst:[1,2],tbl:1,team:[1,12],tech:[],tell:[0,1],templat:[],template_fold:8,template_searchpath:[1,14],templated_command:14,templates_dict:1,templates_ext:1,temporari:[1,2],tenanc:[],tenant:12,term:7,termin:[2,7],test:[],test_env:2,test_hql:1,test_mod:1,test_plugin:8,testabl:5,testview:8,text:[1,3,7,15],textfil:1,than:[1,2,5,14],thei:[0,1,2,3,5,6,8,9,11,14],them:[0,1,2,8,11,14],themselv:2,theori:[4,7],thereof:7,thi:[0,1,2,3,4,5,7,8,11,12,13,14,15],thin:1,thing:[1,2,4,14],think:[5,14],third:7,this_dag_will_be_discov:2,those:[1,2,3,7,12],though:[2,5],thousand:2,thread:1,three:[2,3],threshold:8,thrift:1,through:[1,2,6,7,11,13,14],throughout:1,ticket:[0,12],tighten:12,time:[1,2,3,4,8,11,12,14,15],timedelta:[1,2,4,11
 ,14],timedeltasensor:[1,4],timefram:1,timeout:[0,1,2],timesensor:1,timestamp:[1,2,11],tip:[2,10],titl:7,tls_ca_cert:1,tls_client_cert:1,tls_client_kei:1,tls_hostnam:1,tls_ssl_version:1,tmp:[1,2],tmp_dir:1,tmp_file:1,todai:1,togeth:[1,2],token:1,toler:[1,2],tomorrow:1,tomorrow_d:1,tomorrow_ds_nodash:1,too:[1,2,11],tool:[1,2,6,8],toolbox:8,top:[],topic:3,tornado:0,tort:7,tour:[13,14],track:[2,7,14],tracker:1,trade:7,trademark:7,tradit:1,tradition:2,train:7,transact:1,transfer:[1,2,7],transform:7,translat:[1,7],transpar:15,travers:1,treat:[1,2],tree:[],tree_view:1,tri:[1,12],trick:10,trigger:[],trigger_dag:[0,11],trigger_dag_id:1,trigger_rul:[1,2],triggerdagrunoper:[1,10],triggerrul:1,troubl:6,troubleshoot:[5,15],truncat:1,truth:1,ts_dim:1,ts_nodash:1,tsv:1,tty:1,tunnel:[1,12],tupl:1,turn:[2,12],tutori:[],twice:1,two:[1,2,14],type:[0,1,2,4,7,9,12],typic:[1,2],ubuntu:3,uid:12,ultim:[1,14],unassign:2,unavail:3,unblock:11,undefin:12,under:[1,2,3,7,8,10],underli:8,underscor:3,understand:[1
 ,4,8,14,15],undesir:11,unexpect:2,unexpectedli:2,union:7,uniqu:[1,14],unit:[3,5,12],unittest:1,unix:1,unless:[4,7],unpack:[1,2],unpaus:0,unset:1,unsound:[1,2],unsuccess:1,until:[1,2,3],up_for_retri:1,updat:[1,2,4,12],upgrad:0,upgradedb:0,upload:1,upon:[2,3,13],upstream:[0,1,2,4,11],upstream_fail:[1,2],upstream_list:1,upstream_task_id:1,uri:[1,2,3,12],url:[1,3,8,9,12],usag:[0,14],use_beelin:[1,12],user:[0,1,2,3,5,6,12,14],user_defined_macro:1,user_filt:12,user_name_attr:12,usernam:[1,3,12],usr:3,usual:[2,3],utf8:1,utf:1,util:[1,5,13],uuid:1,val:[0,1],valid:[],valu:[0,1,2,3,4,12,14,15],valuecheckoper:1,variabl:[],vast:1,verbal:7,verbos:[1,14],veri:[0,1,2,4,10,13,14],verifi:[2,4],version:[0,1,2,5,6,7],vertia:1,vertica:[1,6],vertica_conn_id:1,vertica_default:1,verticahook:1,verticaoper:1,verticaql:1,verticatohivetransf:1,via:[1,2,12],video:[1,10],view:[],virtualenv:2,visibl:[1,2],visual:[5,8,9,14,15],vocabulari:2,volum:1,wai:[1,2,3,4,6,8,11,12,14,15],wait:[0,1,2],wait_for_downstream:[1,
 4],wait_for_empty_queu:2,walk:14,want:[1,2,3,4,8,11,12,14],warranti:7,wasn:4,wasnt:1,watch:[1,3,4],weapon:9,web:[],webhdfs_conn_id:1,webhdfs_default:1,webhdfshook:1,webhdfssensor:1,webserv:[0,3,4,12,13,14],week:11,weekli:11,weight:1,well:[1,2,3,4,11],were:1,what:[],whatev:[2,3,6],when:[1,2,3,4,5,8,9,11,12,14],whenev:1,where:[1,2,3,4,7,11,12,14,15],wherev:[1,7],whether:[1,7,11],which:[0,1,2,3,4,7,11,12,13,14],whitelist:12,who:8,whole:[1,2,7],whom:7,whose:11,why:[],wiedmer:10,wiki:10,wildcard:1,wildcard_kei:1,wildcard_match:1,win:1,window:11,winter:10,wire:2,within:[0,1,2,3,4,7],without:[0,1,2,7,11,12],won:[1,4,6],word:[2,3,11,15],work:[0,1,2,3,4,5,7,8,9,11,12,13],worker:[0,1,2,3,4,5,6,8,12,14],worker_timeout:0,workerclass:0,workflow:[],workload:8,worldwid:7,worri:14,worth:1,would:[1,2,3,4,5,11,12,14],wouldn:[1,4],wrap:14,wrapper:1,write:[3,5,7,8,9,14],written:[1,7,14],wrong:14,www:[1,7],x01:1,xcom:[],xcom_al:1,xcom_pul:[1,2],xcom_push:[1,2],xcom_return_kei:1,xml:12,xst:12,yaml:2,year
 :[1,11],yearli:11,yellow:1,yesterdai:1,yesterday_d:1,yesterday_ds_nodash:1,yet:[1,2,12,15],you:[1,2,3,4,5,6,7,8,9,11,12,13,14,15],your:[],yourself:1,youtu:[],youtub:1,yum:6,yvinec:10,yyyi:[0,1,2],yyyymmdd:1,zero:1,zip:[1,2],zip_dag:2,zip_dag_cont:2,zoom:2},titles:["Command Line Interface","API Reference","Concepts","Configuration","FAQ","Apache Airflow (incubating) Documentation","Installation","License","Plugins","Data Profiling","Project","Scheduling & Triggers","Security","Quick Start","Tutorial","UI / Screenshots"],titleterms:{"default":[1,2,14],"function":2,"import":14,addit:2,adhoc:9,airflow:[5,6,8],apach:5,api:1,argument:[2,14],assign:2,authent:12,backend:3,backfil:14,baseoper:1,basesensoroper:1,beyond:5,bitshift:2,branch:2,build:8,celeri:3,chart:[9,15],cluster:2,code:15,command:[0,14],committ:10,commun:[1,3],composit:2,concept:2,configur:3,connect:[2,3],content:5,context:[2,15],contribut:[1,3],core:2,dag:[2,11,14,15],data:9,definit:14,depend:14,document:[2,5],durat:15,en
 abl:12,enterpris:12,exampl:[8,14],executor:1,extern:11,extra:6,faq:4,file:14,form:9,gantt:15,get:6,ghe:12,github:12,graph:15,histori:10,hook:[1,2],horizon:5,idea:2,incub:5,instal:6,instanc:[2,15],instanti:14,integr:3,interfac:[0,8],jinja:[2,14],keep:11,kerbero:12,ldap:12,licens:7,limit:12,line:[0,14],link:10,log:3,macro:1,manag:2,menu:15,meso:3,metadata:14,mind:11,model:1,modul:14,multi:12,next:[13,14],note:2,oper:[1,2],option:3,out:3,own:12,packag:[2,6],password:12,pipelin:14,plugin:8,polici:2,pool:2,principl:5,profil:9,project:10,queri:9,queue:2,quick:13,recap:14,refer:1,resourc:10,roadmap:10,roll:12,rule:2,run:[11,14],scale:3,schedul:11,scope:2,screenshot:[9,15],script:14,secur:12,set:[3,12,14],sla:2,start:13,subdag:2,systemd:3,task:[2,14,15],templat:[2,14],tenanc:12,test:14,top:8,tree:15,trigger:[2,11],tutori:14,undead:2,upstart:3,valid:14,variabl:[1,2,15],view:15,web:12,what:[8,13,14],why:8,workflow:2,xcom:2,your:12,zombi:2}})
\ No newline at end of file
+Search.setIndex({envversion:47,filenames:["cli","code","concepts","configuration","faq","index","installation","license","plugins","profiling","project","scheduler","security","start","tutorial","ui"],objects:{"airflow.contrib":{hooks:[1,1,0,"-"],operators:[1,1,0,"-"]},"airflow.contrib.executors.mesos_executor":{MesosExecutor:[1,0,1,""]},"airflow.contrib.hooks":{BigQueryHook:[1,0,1,""],CloudantHook:[1,0,1,""],FTPHook:[1,0,1,""],GoogleCloudStorageHook:[1,0,1,""],SSHHook:[1,0,1,""],VerticaHook:[1,0,1,""]},"airflow.contrib.hooks.BigQueryHook":{get_conn:[1,2,1,""],get_pandas_df:[1,2,1,""],get_service:[1,2,1,""],insert_rows:[1,2,1,""]},"airflow.contrib.hooks.CloudantHook":{db:[1,2,1,""]},"airflow.contrib.hooks.FTPHook":{close_conn:[1,2,1,""],create_directory:[1,2,1,""],delete_directory:[1,2,1,""],delete_file:[1,2,1,""],describe_directory:[1,2,1,""],get_conn:[1,2,1,""],list_directory:[1,2,1,""],retrieve_file:[1,2,1,""],store_file:[1,2,1,""]},"airflow.contrib.hooks.GoogleCloudStorageHook":
 {download:[1,2,1,""],get_conn:[1,2,1,""],upload:[1,2,1,""]},"airflow.contrib.hooks.SSHHook":{Popen:[1,2,1,""],check_output:[1,2,1,""],tunnel:[1,2,1,""]},"airflow.contrib.hooks.VerticaHook":{get_conn:[1,2,1,""]},"airflow.contrib.hooks.gcs_hook":{GoogleCloudStorageHook:[1,0,1,""]},"airflow.contrib.operators":{QuboleOperator:[1,0,1,""],SSHExecuteOperator:[1,0,1,""],VerticaOperator:[1,0,1,""],VerticaToHiveTransfer:[1,0,1,""]},"airflow.contrib.operators.bigquery_operator":{BigQueryOperator:[1,0,1,""]},"airflow.contrib.operators.bigquery_to_gcs":{BigQueryToCloudStorageOperator:[1,0,1,""]},"airflow.contrib.operators.gcs_download_operator":{GoogleCloudStorageDownloadOperator:[1,0,1,""]},"airflow.contrib.operators.hipchat_operator":{HipChatAPIOperator:[1,0,1,""],HipChatAPISendRoomNotificationOperator:[1,0,1,""]},"airflow.executors":{CeleryExecutor:[1,0,1,""],LocalExecutor:[1,0,1,""],SequentialExecutor:[1,0,1,""]},"airflow.hooks":{DbApiHook:[1,0,1,""],DruidHook:[1,0,1,""],HiveCliHook:[1,0,1,"
 "],HiveMetastoreHook:[1,0,1,""],HiveServer2Hook:[1,0,1,""],HttpHook:[1,0,1,""],MsSqlHook:[1,0,1,""],MySqlHook:[1,0,1,""],PostgresHook:[1,0,1,""],PrestoHook:[1,0,1,""],S3Hook:[1,0,1,""],SqliteHook:[1,0,1,""],WebHDFSHook:[1,0,1,""]},"airflow.hooks.DbApiHook":{bulk_dump:[1,2,1,""],bulk_load:[1,2,1,""],get_conn:[1,2,1,""],get_cursor:[1,2,1,""],get_first:[1,2,1,""],get_pandas_df:[1,2,1,""],get_records:[1,2,1,""],insert_rows:[1,2,1,""],run:[1,2,1,""]},"airflow.hooks.DruidHook":{construct_ingest_query:[1,2,1,""],get_conn:[1,2,1,""],load_from_hdfs:[1,2,1,""]},"airflow.hooks.HiveCliHook":{load_file:[1,2,1,""],run_cli:[1,2,1,""],test_hql:[1,2,1,""]},"airflow.hooks.HiveMetastoreHook":{check_for_partition:[1,2,1,""],get_databases:[1,2,1,""],get_metastore_client:[1,2,1,""],get_partitions:[1,2,1,""],get_table:[1,2,1,""],get_tables:[1,2,1,""],max_partition:[1,2,1,""],table_exists:[1,2,1,""]},"airflow.hooks.HiveServer2Hook":{get_pandas_df:[1,2,1,""],get_records:[1,2,1,""]},"airflow.hooks.HttpHook":
 {get_conn:[1,2,1,""],run:[1,2,1,""],run_and_check:[1,2,1,""]},"airflow.hooks.MsSqlHook":{get_conn:[1,2,1,""]},"airflow.hooks.MySqlHook":{bulk_load:[1,2,1,""],get_conn:[1,2,1,""]},"airflow.hooks.PrestoHook":{get_conn:[1,2,1,""],get_first:[1,2,1,""],get_pandas_df:[1,2,1,""],get_records:[1,2,1,""],run:[1,2,1,""]},"airflow.hooks.S3Hook":{check_for_bucket:[1,2,1,""],check_for_key:[1,2,1,""],check_for_prefix:[1,2,1,""],check_for_wildcard_key:[1,2,1,""],get_bucket:[1,2,1,""],get_conn:[1,2,1,""],get_key:[1,2,1,""],get_wildcard_key:[1,2,1,""],list_keys:[1,2,1,""],list_prefixes:[1,2,1,""],load_file:[1,2,1,""],load_string:[1,2,1,""]},"airflow.hooks.SqliteHook":{get_conn:[1,2,1,""]},"airflow.hooks.WebHDFSHook":{check_for_path:[1,2,1,""],get_conn:[1,2,1,""],load_file:[1,2,1,""]},"airflow.macros":{ds_add:[1,3,1,""],ds_format:[1,3,1,""],hive:[1,1,0,"-"],integrate_plugins:[1,3,1,""],random:[1,3,1,""]},"airflow.macros.hive":{closest_ds_partition:[1,3,1,""],max_partition:[1,3,1,""]},"airflow.models":
 {BaseOperator:[1,0,1,""],Connection:[1,0,1,""],DAG:[1,0,1,""],DagBag:[1,0,1,""],TaskInstance:[1,0,1,""]},"airflow.models.BaseOperator":{clear:[1,2,1,""],dag:[1,4,1,""],detect_downstream_cycle:[1,2,1,""],downstream_list:[1,4,1,""],execute:[1,2,1,""],get_direct_relatives:[1,2,1,""],get_flat_relatives:[1,2,1,""],get_task_instances:[1,2,1,""],has_dag:[1,2,1,""],on_kill:[1,2,1,""],post_execute:[1,2,1,""],pre_execute:[1,2,1,""],prepare_template:[1,2,1,""],render_template:[1,2,1,""],render_template_from_field:[1,2,1,""],run:[1,2,1,""],schedule_interval:[1,4,1,""],set_downstream:[1,2,1,""],set_upstream:[1,2,1,""],upstream_list:[1,4,1,""],xcom_pull:[1,2,1,""],xcom_push:[1,2,1,""]},"airflow.models.Connection":{extra_dejson:[1,4,1,""]},"airflow.models.DAG":{add_task:[1,2,1,""],add_tasks:[1,2,1,""],cli:[1,2,1,""],concurrency_reached:[1,4,1,""],crawl_for_tasks:[1,2,1,""],create_dagrun:[1,2,1,""],filepath:[1,4,1,""],folder:[1,4,1,""],get_template_env:[1,2,1,""],is_paused:[1,4,1,""],latest_executi
 on_date:[1,4,1,""],run:[1,2,1,""],set_dependency:[1,2,1,""],sub_dag:[1,2,1,""],subdags:[1,4,1,""],tree_view:[1,2,1,""]},"airflow.models.DagBag":{bag_dag:[1,2,1,""],collect_dags:[1,2,1,""],dagbag_report:[1,2,1,""],get_dag:[1,2,1,""],kill_zombies:[1,2,1,""],process_file:[1,2,1,""],size:[1,2,1,""]},"airflow.models.TaskInstance":{are_dependencies_met:[1,2,1,""],are_dependents_done:[1,2,1,""],clear_xcom_data:[1,2,1,""],command:[1,2,1,""],current_state:[1,2,1,""],error:[1,2,1,""],evaluate_trigger_rule:[1,2,1,""],is_premature:[1,2,1,""],is_queueable:[1,2,1,""],is_runnable:[1,2,1,""],key:[1,4,1,""],pool_full:[1,2,1,""],ready_for_retry:[1,2,1,""],refresh_from_db:[1,2,1,""],run:[1,2,1,""],xcom_pull:[1,2,1,""],xcom_push:[1,2,1,""]},"airflow.operators":{BashOperator:[1,0,1,""],BranchPythonOperator:[1,0,1,""],DummyOperator:[1,0,1,""],EmailOperator:[1,0,1,""],ExternalTaskSensor:[1,0,1,""],GenericTransfer:[1,0,1,""],HdfsSensor:[1,0,1,""],Hive2SambaOperator:[1,0,1,""],HiveOperator:[1,0,1,""],HivePa
 rtitionSensor:[1,0,1,""],HiveToDruidTransfer:[1,0,1,""],HiveToMySqlTransfer:[1,0,1,""],HttpSensor:[1,0,1,""],MetastorePartitionSensor:[1,0,1,""],MsSqlOperator:[1,0,1,""],MsSqlToHiveTransfer:[1,0,1,""],MySqlOperator:[1,0,1,""],MySqlToHiveTransfer:[1,0,1,""],PostgresOperator:[1,0,1,""],PrestoCheckOperator:[1,0,1,""],PrestoIntervalCheckOperator:[1,0,1,""],PrestoValueCheckOperator:[1,0,1,""],PythonOperator:[1,0,1,""],S3KeySensor:[1,0,1,""],S3ToHiveTransfer:[1,0,1,""],ShortCircuitOperator:[1,0,1,""],SimpleHttpOperator:[1,0,1,""],SlackAPIOperator:[1,0,1,""],SlackAPIPostOperator:[1,0,1,""],SqlSensor:[1,0,1,""],TimeSensor:[1,0,1,""],TriggerDagRunOperator:[1,0,1,""],WebHdfsSensor:[1,0,1,""]},"airflow.operators.BashOperator":{execute:[1,2,1,""]},"airflow.operators.SlackAPIOperator":{construct_api_call_params:[1,2,1,""],execute:[1,2,1,""]},"airflow.operators.docker_operator":{DockerOperator:[1,0,1,""]},"airflow.operators.sensors":{BaseSensorOperator:[1,0,1,""]},airflow:{executors:[1,1,0,"-"],h
 ooks:[1,1,0,"-"],macros:[1,1,0,"-"],models:[1,1,0,"-"],operators:[1,1,0,"-"]}},objnames:{"0":["py","class","Python class"],"1":["py","module","Python module"],"2":["py","method","Python method"],"3":["py","function","Python function"],"4":["py","attribute","Python attribute"]},objtypes:{"0":"py:class","1":"py:module","2":"py:method","3":"py:function","4":"py:attribute"},terms:{"00am":1,"01t23":11,"10gb":1,"10pm":2,"128m":1,"2am":1,"5gb":1,"__class__":2,"__doc__":2,"__init__":[1,2],"__name__":[2,8],"_cmd":3,"_host":12,"abstract":[1,2,5,8],"boolean":[1,2],"byte":[1,4],"case":[1,2,4,11,12,14],"char":1,"class":[0,1,2,6,7,8,14],"default":0,"export":[1,13],"final":1,"float":1,"function":1,"import":[1,2,4,5,6,7,8,11,12],"int":1,"long":1,"new":[1,2,4,8,11,12],"return":[0,1,2,8],"short":1,"static":[1,5,8],"super":[1,12],"switch":[3,12],"true":[1,2,4,12,14],"try":[1,2,3],"var":1,"while":[1,2,4,5,7,11,13,14,15],abbrevi:2,abil:2,abl:[3,4,12,13,14],about:[1,2,3,7,10,11,14],abov:[2,3,7,12,15],abs
 enc:2,absolut:2,accept:[1,5,7],access:[0,1,2,3,8,12],access_logfil:0,access_token:15,accord:12,accordingli:2,account:[1,12],acquir:1,across:[3,15],act:[1,2,7,14],action:[1,15],activ:[1,2,4,8,11,12,14],actual:[1,2,4,14],acycl:[1,2,5],add:[1,2,3,7,12],add_task:1,addendum:7,addit:1,addition:12,addprinc:12,adhoc:[0,1,5],adjust:[3,12],admin:[2,3,4,8,9,12,13],admin_view:8,advanc:[1,14],advantag:12,advis:[1,7],affect:2,after:[1,2,4,11],afterward:1,against:[1,2,4,7,12,14],agari:10,aggregate_db_message_job:2,aggregate_db_message_job_cmd:2,ago:4,agre:7,agreement:[2,7],airbnb:[1,2,10,14],airflow:[0,1,2,3,4],airflow__:3,airflow__core__sql_alchemy_conn:3,airflow_config:3,airflow_conn_:[2,3],airflow_conn_postgres_mast:2,airflow_hom:[3,4,8,13],airflow_set:2,airflow_tmp_dir:1,airflowignor:1,airflowplugin:8,airflowtestplugin:8,aiyzngq:1,alchemi:3,alert:[1,2,8],all:[0,1,2,3,4,5,6,7,11,12,14],all_db:6,all_don:[1,2],all_fail:[1,2],all_success:[1,2],alleg:7,alloc:1,allow:[0,1,2,3,4,5,8,9,11,12,13,14,15]
 ,allow_large_result:1,allowed_st:1,allowed_team:12,allowinsert:1,alon:7,along:[1,4,7,14],alongsid:[1,7,11],alreadi:1,alright:14,also:[1,2,3,4,6,7,8,11,12,14],alter:[1,2,3,4,12],altern:[1,11],altert:1,although:12,alwai:[1,2,15],amazon:3,amount:[1,2],analys:15,analyz:2,anand:10,anchor:1,ani:[1,2,3,4,7,14,15],annot:7,announc:10,anomali:8,anoth:[1,2,4,10],ansibl:3,anyon:12,anyth:[2,14],anytim:2,anywher:[1,2],aoen:10,apach:1,api:0,api_call_param:1,api_kei:15,api_param:1,api_vers:1,apikei:15,apiv2:1,app:1,app_id:1,appar:1,appear:[1,2,7],append:3,appendix:7,appli:[1,2,6,7,11],applic:[1,7,8,9,12],appropri:[2,7,12],arbitrari:[2,5],architectur:5,archiv:[1,7],archive1:1,archive2:1,are_dependencies_met:1,are_dependents_don:1,aren:[1,2],arg:1,argument:[0,1],aris:7,around:[1,5,14],arrai:[1,5,8],arthur:10,artwr:10,ascii:1,asf:5,assert:7,asset:1,assign:1,associ:[1,2,7,11],assum:[1,2,3,7,14],asterisk:12,async:6,atom:2,attach:[1,2,7],attach_to:1,attack:12,attempt:[0,1],attent:[1,4],attr:1,attribut:[0
 ,1,2,7],audit:8,auth:[1,12],auth_backend:12,auth_token:1,authen:12,authent:[1,2,5,6],authmechan:1,author:[1,3,5,7,12,14,15],authorship:7,autocommit:1,automat:[1,2,3],avail:[1,2,3,7,8,9],averag:1,avoid:2,awai:1,awar:2,azkaban:5,back:[1,4,8],backend:1,backfil:[0,1,4,5,11,13],background:[1,14],backup:3,bag:1,bag_dag:1,balanc:2,bar:[1,2],bare:14,base64:4,base:[1,2,3,4,7,8,11],base_executor:[1,8],base_hook:[1,8],base_url:1,basedn:12,baseexecutor:[1,8],basehook:[1,2,8],baseview:8,bash:[1,2,14],bash_1:2,bash_command:[1,2,14],bash_command_to_run:3,bash_queu:14,bashoper:[1,2,14],basi:7,basic:[2,6,8,9,14],bcrypt:12,beauchemin:10,becaus:[1,2],becom:[1,2,5,8,14],beelin:1,been:[1,2,3,7,11,12],befor:[0,1,2,4,12],begin:[2,11],behalf:7,behav:1,behavior:[1,2],behind:[1,3,6,11],belong:12,below:[1,7,8,13,14],benefici:7,best:[1,10],better:14,between:[1,2,14],beyond:[1,4],bigqueri:1,bigquery_conn_id:1,bigquery_default:1,bigquery_oper:1,bigquery_to_gc:1,bigqueryhook:1,bigqueryoper:1,bigquerytocloudstorag
 eoper:1,bin:2,bind:7,bind_password:12,bind_us:12,bit:3,blob:[1,14],block:[1,2,14,15],blog:[7,10],blueprint:8,bodi:1,boilerpl:7,bolk:10,bolkedebruin:10,bonu:1,bool:1,boot:3,both:[1,4],bother:14,boto:1,bottleneck:15,boundary_queri:1,box:[1,3,13],bql:1,bracket:[7,14],branch:1,branch_fals:2,branchpythonoper:[1,2],broker:[0,3],broker_api:0,broker_url:3,brought:10,brows:[2,4],bruin:10,bucket:[1,3],bucket_kei:1,bucket_nam:1,buffer:1,bug:3,build:[1,2,3,5],built:[1,2,3,5,8,9,14],bulk:[4,15],bulk_dump:1,bulk_load:1,bump:2,bundl:1,burn:0,but_this_dag_will_not:2,c12318391:1,cacert:12,cach:12,call:[1,2,3,14],callabl:1,callback:12,can:[1,2,3,4,5,6,8,9,11,12,13,14,15],cannot:[1,2,7,14],capabl:8,capac:2,card:1,care:[0,3],carri:[2,7],cast:1,cat:1,categori:8,caus:[4,7],celeri:[0,1,2],celery_result_backend:3,celeryexecutor:[1,2,3,4,6,11],cell:1,central:2,cert:1,certain:[1,2],certif:1,cfg:[1,2,3,4,11,12,13,14],chain:2,chang:[3,5,7,11,12,13,14],channel:[1,10],charact:7,character:2,charg:7,charset:1,char
 t:[5,8],chat:10,check:[0,1,2,4],check_for_bucket:1,check_for_kei:1,check_for_partit:1,check_for_path:1,check_for_prefix:1,check_for_wildcard_kei:1,check_head:1,check_oper:1,check_output:1,checkoper:1,chef:[3,10],child:[1,2],child_dag:2,child_dag_nam:2,chmod:12,choic:[0,1,14],choos:[1,2,7],chri:[5,10],circuit:1,claim:7,clariti:5,claus:1,clean:1,cleanup:1,clear:[0,1,2,4,11,15],clear_xcom_data:1,cli:[1,2,3,4,11],click:[4,12,14,15],client:[1,6,12],client_id:12,client_secret:12,close:[1,4,12],close_conn:1,closest:1,closest_ds_partit:1,cloud:[1,3,6],cloudant:[1,6],cloudant_conn_id:1,cloudant_default:1,cloudanthook:1,cluster:1,cluster_label:1,cmd:1,cmdline:1,code:[0,1,2,3,5,7,8,14],col:1,collabor:5,collect:[1,2,8],collect_dag:1,color:1,column:1,com:[1,2,12,14],combin:[2,7],come:[1,11],comma:[0,1,2],command_typ:1,comment:7,commerci:7,commit:[1,10,12],commit_everi:1,committ:5,common:[2,3,4,7,10,14],commonli:1,compani:8,compar:5,compil:[4,7],complet:[1,2,3,5],complex:[1,2,5,8],compli:7,compli
 anc:7,complic:14,compon:8,compos:[2,12,14],comprehens:15,compress:1,comput:[1,7],conceptu:2,concern:2,concur:4,concurr:[0,1,4],concurrency_reach:1,condit:[1,2,6,7],conf:[0,1,3],config:[8,12],configur:[1,2],confirm:[0,4,11],confus:[1,2,4],conjunct:[0,2,13],conn_id:[1,2,3],conn_typ:1,connect:1,connect_timeout:1,connecton:9,consequenti:7,consid:[1,2,3],consider:1,consist:[1,2,5,7],conspicu:7,constant:[1,2],constitu:2,constitut:7,constraint:1,constru:7,construct:1,construct_api_call_param:1,construct_ingest_queri:1,constructor:[1,14],contact:[3,7],contain:[1,2,3,7,11,12,14,15],content:[1,2],context:1,continu:[1,5],contract:7,contrib:[1,2,12],contributor:[7,10],contributori:7,control:[1,2,7,15],conveni:[1,2],convent:2,convers:7,cool:1,coordin:1,copi:[1,3,7,12],copyright:[7,12],core:1,correct:2,correpsond:2,correspond:[1,2,3,4],could:[1,2,14],count:1,counterclaim:7,cours:12,cover:11,cpu:[1,8],craft:1,crawl:[1,11],crawl_for_task:1,creat:[1,2,3,4,8,9,11,12,13,14,15],create_dagrun:1,create_d
 irectori:1,creation:1,credenti:12,credit:12,criccomini:10,criteria:[1,2],criterion:1,critic:[1,2],cron:[1,4,11],cross:[2,7,14],crt:12,crypto:[3,4,6],csv:1,curli:14,current:[1,15],current_st:1,cursor:1,custom:8,customari:7,cycl:[1,2,14],daemon:[0,1,3],dag:[0,1],dag_1:2,dag_2:2,dag_fold:[1,2],dag_id:[0,1,2,14],dag_md:2,dag_run:1,dag_run_obj:1,dag_stat:0,dagbag:[0,1],dagbag_report:1,dagfil:2,dagrun:[0,1,4],dagrun_timeout:1,dags_fold:3,dai:[1,11,13,14],daili:[1,4,11],damag:7,dan:[1,10],data:[1,2,5,6,8],data_profiler_filt:12,databas:[0,1,2,3,5,6,8,9,13,14],datafram:1,dataset:1,datasourc:1,date:[0,1,2,7,11,14],date_filter_column:1,datestamp:1,datetim:[1,2,4,11,14],dateutil:1,davydov:10,days_back:1,db_tabl:1,db_tap_id:1,db_update_kei:1,db_update_mod:1,dbapi_hook:1,dbapihook:1,dbexportcmd:1,dbimportcmd:1,dbtap_id:1,dbtapquerycmd:1,deal:4,debug:[0,1,14],decid:[1,3],decis:5,declar:1,deep:1,def:[1,2,8],default_arg:[1,2,4,14],default_login:12,default_queu:2,defend:7,defer:2,deferred_op:2,defin:
 [1,2,3,4,5,7,8,11,12,14],definit:[2,5,7,8],del:1,delai:[1,4],delegate_to:1,delet:[1,2,11,15],delete_directori:1,delete_fil:1,deliber:[1,7],delimit:[0,1,2],demonstr:2,depend:[0,1,2,3,4,5,6,11],depends_on_past:[0,1,2,4,14],deploi:8,deploy:8,depth:14,deriv:[1,3,7,8],descend:2,describ:[1,2,7],describe_directori:1,descript:[1,2,7],deseri:[0,1,2],deserialize_json:2,design:[2,7,11],desir:[1,14],destin:1,destination_cloud_storage_uri:1,destination_conn_id:1,destination_dataset_t:1,destination_filepath:1,destination_t:1,detail:[1,2,4,12,15],detect:[1,8],detect_downstream_cycl:1,determin:[1,2,7],dev:[3,6,10],devel:6,devel_hadoop:6,develop:[0,1,12,14],deviat:1,dialog:11,dict:[0,1,2],dictionari:[1,2,14],did:14,die:2,differ:[1,2,7,8,14,15],dimens:1,direct:[1,2,3,5,7],directli:[1,2,4,9],directori:[0,1,2,3,12,14],dirti:13,disabl:1,disclaim:7,discuss:7,disk:1,displai:[3,7,11,14],disregard:[1,14],distinct:1,distribut:[1,2,6,7],do_pickl:0,doc:[1,2,4,14],doc_json:2,doc_md:2,doc_rst:2,doc_yaml:2,docker
 :[1,10],docker_oper:1,docker_url:1,dockeroper:[1,2],document:1,doe:[0,1,2,3,4,5,6,7,12,14],does_not_exist:1,doesn:[1,2,14],domain:12,don:[1,2,3,6,7,11],done:[1,2,11,12,14],donot_pickl:[0,1],door:2,dot:[1,2],doubl:[1,3,14],down:[0,2],download:[1,3],downstream:[0,1,2,11],downstream_list:1,downstream_task_id:1,drive:3,driven:[1,8],driver:3,drop:[1,8],druid:[1,6],druid_datasourc:1,druid_ingest_conn_id:1,druid_ingest_default:1,druid_query_conn_id:1,druid_query_default:1,druidhook:1,dry:0,dry_run:0,ds_add:[1,14],ds_format:1,ds_nodash:1,dubiou:1,dummi:[1,2,8],dummy_1:2,dummy_oper:2,dummy_task:2,dummyoper:[1,2],dump:1,durat:5,dure:[3,15],dynam:[1,2,4,5],each:[1,2,4,7,10,11,14],earlier:14,easi:[2,3,5,8,9,12,15],easier:[1,7],easiest:6,easili:[1,2,5,9,14],echo:[2,14],ecosystem:8,edit:[3,15],editori:7,effect:[1,2],effici:1,effort:5,egg:3,either:[1,2,3,7,12,13],elabor:7,elaps:1,elast:1,electron:7,eleg:5,element:3,els:13,elsewher:2,email:[1,2,10,12,14],email_on_failur:[1,14],email_on_retri:[1,14]
 ,emailoper:[1,2],emit:[2,14],empti:[1,2],enabl:[1,2,3,5,6],enclos:7,encod:[1,4],encrypt:[1,3,4,6,12],encrypt_s3_log:3,end:[1,7,11],end_dat:[0,1,14],endors:5,endpoint:1,enforc:[2,4],engin:[1,4,5],enterpris:5,entir:[0,4],entiti:7,entri:[1,11],env:[1,2],environ:[1,2,3,5,6,9,11,14,15],ep_data_pipeline_db_msg_agg:2,equival:[2,6,14],era:4,error:[0,1],error_logfil:0,especi:[2,4,14],essenti:1,etc:[1,2,3,12],eval:1,evalu:[1,2,4,11,14],evaluate_trigger_rul:1,even:[1,2,7,9],event:[2,7,14],eventlet:0,ever:1,everi:[1,2,11,12,14],everyon:14,everyth:[14,15],exactli:[1,15],exampl:[1,2,3,5,7],example1:13,example_bash_oper:13,example_dag:[2,14],example_team_1:12,example_team_2:12,exce:1,except:[1,2,4,7,14],exchang:[2,5],exclud:[0,7],exclude_subdag:0,exclus:[7,11],execut:[0,1,2,3,4,5,7,11,14],execution_d:[0,1,2,14],execution_delta:1,execution_timeout:[1,2],exercis:7,exhaust:3,exist:[0,1,2,3,4,14],exit:[0,12],expect:[1,2,4,5],expir:1,explain:1,explan:14,explicit:5,explicit_op:2,explicitli:[1,2,7,12,14]
 ,export_dir:1,export_format:1,expos:[1,2,3,8,12],express:[1,4,7,11],ext:1,extend:5,extens:[1,5],extern:[1,2,5,8,10],external_dag_id:1,external_task_id:1,external_trigg:1,externaltasksensor:1,extra:[1,2,5],extra_dejson:1,extra_opt:1,extract:1,extract_queri:1,fact:2,factori:2,fail:[0,1,2,3,4,11,14,15],failur:[1,3,4,7],fairli:13,fall:10,fals:[0,1,3,11,14],familiar:[2,14],faq:1,fast:13,fault:[1,2],featur:[1,2,3,6,8,14,15],feb:12,fee:7,fernet:4,fernet_kei:4,fetch:[1,3],few:[1,3,9,13,14],field:[1,7],field_delimit:1,field_dict:1,fields_terminated_bi:1,fifti:7,figur:[1,2,11],file1:1,file2:1,file:[0,1,2,3,5,7,8,12,13],filenam:[1,2],filepath:1,filestatu:1,filesystem:3,fill:[1,2,9,12],filter:[0,1,2,12],filter_by_own:12,find:[1,3,4,14,15],fine:14,finish:1,fire:[1,2,3],first:[0,1,2,3,4,10,11,13,14],fit:[1,5,7],fix:[1,11],flag:[1,11],flag_upstream_fail:1,flask:[0,8,9],flask_admin:8,flask_blueprint:8,flask_login:12,flat:1,flavor:8,flexibl:[1,14],flower:[0,3],folder:[1,3,8,11,13,14],follow:[1,2,3,5
 ,7,12,14],foo:[1,2],foo_dag_id:4,foobar:3,forc:[0,1],force_pul:1,foreground:0,foreign:1,form:[2,5,7],format:[1,2,3,7],forward:[1,4],found:[1,3,12],foundat:[5,7,10,13],four:2,framework:[3,8],free:[2,7],frm:1,from:[0,1,2,3,4,5,7,8,9,10,11,12,13,14,15],ftp:1,ftp_conn_id:1,ftp_default:1,ftphook:1,full:[1,10],full_filepath:1,fulli:[4,5,12],functiontyp:1,fundament:14,further:[5,13,14],futur:[1,11],galaxi:8,gantt:5,garag:2,gate:12,gcp_api:6,gcp_api_base_hook:1,gcs_download_oper:1,gcs_hook:1,gender:1,gener:[0,1,2,5,7,8,12,14,15],generictransf:1,get:[0,1,2,3,4,5],get_bucket:1,get_conn:1,get_connect:2,get_cursor:1,get_dag:1,get_databas:1,get_direct_rel:1,get_first:1,get_flat_rel:1,get_kei:1,get_metastore_cli:1,get_pandas_df:1,get_partit:1,get_partitions_by_filt:1,get_record:1,get_servic:1,get_tabl:1,get_task_inst:1,get_template_context:1,get_template_env:1,get_wildcard_kei:1,gevent:0,ghe:5,ghe_oauth:12,ghost:1,git:3,github:[1,5,7,10],github_enterpris:12,github_enterprise_auth:12,githubusercon
 t:1,gitignor:1,gitter:10,give:[2,7],given:[1,2],glanc:15,global:[1,2,4],goal:14,goe:[1,4,14],good:2,goodby:2,goodwil:7,googl:[1,3,6],google_cloud_storage_conn_id:1,google_cloud_storage_default:1,googlecloudbasehook:1,googlecloudstoragedownloadoper:1,googlecloudstoragehook:1,govern:7,grab:1,grade:3,grai:1,grant:7,graph:[1,2,5],great:[2,3],greater:1,green:1,grossli:7,group:[1,3,12],gunicorn:[0,6],hack:1,had:[1,4],hadoop:[6,10,12],hadoop_dependency_coordin:1,hadoopcmd:1,hand:[12,13],handi:10,handl:[1,2,3],happen:[2,11,14],hard:[1,2],harmless:7,has_dag:1,hash:[1,12],hasn:[1,2],have:[1,2,3,4,5,6,7,8,9,11,12,14],haven:[1,2,14],hdf:[1,2,6],hdfs_conn_id:1,hdfs_default:1,hdfs_path:1,hdfscli:1,hdfssensor:1,head:[13,14],header:1,heartbeat:[1,2],heavier:1,hello:[2,8],help:[0,7,8,12],here:[1,2,3,4,6,8,10,11,13,14,15],herebi:7,herein:7,hex:1,hidden:15,hierarchi:[1,14],high:1,highchart:9,higher:1,highli:3,highlight:14,hipchat:1,hipchat_oper:1,hipchatapioper:1,hipchatapisendroomnotificationoper:1,h
 istori:5,hit:[1,2],hive2mysqloper:1,hive2sambaoper:[1,6],hive:[1,2,3,6,8,12],hive_cli_conn_id:1,hive_cli_default:1,hive_cli_param:1,hive_conn_id:1,hive_t:1,hiveclihook:1,hivecmd:1,hiveconf:1,hiveconf_jinja_transl:1,hivemetastorehook:1,hiveoper:[1,2,3,12],hivepartitionsensor:[1,2],hivesensoroper:1,hiveserver2:1,hiveserver2_conn_id:1,hiveserver2_default:1,hiveserver2hook:1,hivetodruidtransf:1,hivetomysqltransf:1,hold:7,home:13,homogen:3,hope:11,horribl:14,host:[1,3,12],hostnam:[0,1,2,3],hour:[1,2,4,11],hourli:[1,4,11],hous:2,how:[1,2,3,4,7,8,11,14,15],howev:[2,7,12],hql:1,html:[1,8,14],html_content:1,http:[1,2,7,8,12,14],http_conn_id:1,http_default:1,httphook:1,httpoper:2,httpsensor:1,human:1,icon:1,icon_url:1,idea:1,idempot:1,identif:7,identifi:[0,1,7,14,15],ignor:[0,1],ignore_depend:[0,1],ignore_depends_on_past:[0,1],ignore_first_depends_on_past:[0,1],iii:7,imag:1,immedi:[1,2],imper:3,imperson:12,implement:[1,2],impli:[1,7],improv:7,impyla:1,inabl:7,inact:4,inc:7,incident:7,includ:[
 0,1,2,3,7,11,14],include_adhoc:[0,1],include_downstream:1,include_exampl:1,include_prior_d:1,include_queu:1,include_upstream:1,inclus:7,incorpor:7,incur:7,indemn:7,indemnifi:7,independ:[1,2],index:1,indic:[1,2,5,7],indirect:7,individu:[1,7,11,14],infer:[1,2],inferred_op:2,infil:1,infin:5,info:1,inform:[1,2,3,7,11,12,14],infrastructur:[1,5],infring:7,ingest:1,inherit:[1,14],init:3,initctl:3,initdb:[0,3,13],initi:[0,3,12,13],initialis:1,inject:8,inlin:1,input:1,input_format:1,insecur:12,insecurecli:1,insert:[1,2],insert_row:1,insid:[1,2],inspect:[11,13],instal:[1,2,3,4,5],instanc:[0,1],instanti:[1,2,5,11],instati:14,instead:[0,1,4],insteadnhttp:1,institut:7,instruct:2,insur:1,integr:1,integrate_plugin:1,intention:7,inter:2,interact:[1,2,3,8,9],interest:[1,14],intergr:8,intern:0,interpret:[1,2],interv:1,intervalcheckoper:1,intoth:1,introduc:4,intuit:14,invari:[1,2],invok:1,ipython:2,irrevoc:7,is_paus:1,is_prematur:1,is_queueabl:1,is_runn:1,ish:1,isn:[1,2,4],isoformat:1,issu:[1,2,5,7,10
 ],item:2,iter:1,itself:[1,2,12,14],januari:[7,11],jar:1,java:1,jdbc:[1,6],jeremiah:10,jinja2:1,jinja:1,jinja_env:1,jlowin:10,job:[0,1,2,3,4,11,13,15],job_id:1,jobtrack:1,join:[2,10],json:[0,1,2],jupyt:2,just:[0,1,2,4,8,14],kadmin:12,kapil:3,keep:[1,2,5],kei:[0,1,2,3,4,12,15],kerber:[6,12],kerbero:[0,5,6],kernel:1,key_fil:1,keytab:[0,12],keyword:1,kick:[3,11],kill:[1,2],kill_zombi:1,kind:7,know:[3,14],knowledg:8,known:6,known_host:1,kruyk:10,kwarg:1,kwd:1,l447:1,lab:1,label:[1,9],lai:13,lambda:[1,2],land:[1,8],languag:[1,7,14],larg:1,larger:1,last:1,late:15,later:9,latest:[0,1,6,14],latest_d:1,latest_execution_d:1,latin_stat:1,launch:12,law:7,lawsuit:7,layout:14,ldap3:12,ldap:[5,6],ldap_auth:12,ldap_ca:12,lead:[1,2],lean:5,least:[1,2,7],leav:1,led:2,left:[2,12],legaci:4,legal:7,len:1,leo:10,less:1,lesson:10,let:[0,1,2,3,9,14,15],level:[1,2,5,11,12],leverag:[1,2,14],liabil:7,liabl:7,lib:[1,2,3],libffi:3,librari:[1,2,3,5,8,14],libz:2,licens:5,licensor:7,life:2,light:[1,2],lighter:1,lig
 htweight:2,like:[1,2,3,6,8,12,14],limit:[1,2,3,5,7],link:[2,5,7,8],list:[0,1,2,3,4,6,7,8,10,12,14,15],list_dag:[0,4,14],list_directori:1,list_kei:1,list_prefix:1,list_task:[0,4,14],listen:2,lit:1,liter:1,litig:7,live:1,load:[0,1,2],load_fil:1,load_from_hdf:1,load_str:1,local:[0,1,2,3,12,14],local_executor:1,local_full_path_or_buff:1,local_infil:1,local_port:1,localexecutor:[0,1,2,3,11],localhost:2,locat:[0,1,2,3,12,14],lock:1,lock_for_upd:1,log:[0,1],log_fil:0,logfil:0,loggingmixin:1,logic:[1,2,14],login:[1,2,3,12],logist:8,look:[0,1,2,5,8,10,11,14],loss:7,lot:11,lowin:10,machin:[1,2,3,12],made:[1,2,7],mai:[1,2,3,4,7,11,14],mail:[7,10],main:[1,2,4,8],main_dag:2,maintain:[1,5,10],make:[1,2,3,4,5,7,12,14,15],malfunct:7,malik:3,man:6,manag:1,mani:[0,1,2,4,8,14,15],manner:5,manual:[1,2,3,11],map:1,mapr:1,mark:[0,1,2,7,11],mark_success:[0,1,11],markdown:2,markup:1,master:[1,2,14],match:[0,1,2],matter:1,max:[1,10],max_active_run:[1,4],max_partit:1,maxim:10,maxime_beauchemin:0,maximum:1,ma
 yb:2,mean:[1,2,3,4,7,12,14],meaning:1,meant:1,mechan:[1,7,12],media:7,medium:7,meet:[1,7],mem_limit:1,member:12,memberof:12,memori:1,menu:[2,4,5,8,13],menu_link:8,menulink:8,merchant:7,mere:7,merg:14,meso:1,mesos_executor:1,mesosexecutor:[1,3,11],messag:[1,2,5],message_format:1,met:[1,3,4,11],metadata:[0,1,2,3,4,5,6,8,11],metastor:1,metastore_conn_id:1,metastore_default:1,metastore_mysql:1,metastorepartitionsensor:1,method:[1,2],metric:[1,8],metric_spec:1,metrics_threshold:1,microsoft:[1,6],midnight:[4,11],might:[1,2,4,11,14],migrat:12,mime:1,mime_typ:1,min:4,mind:[2,5],minimum:[1,6,11],minut:[2,4,11,14],mismatch:2,miss:[1,2],mistercrunch:10,mix:14,mkdir:2,mlsd:1,mode:[0,1,14],modif:7,modifi:[1,2,3,7,9],modul:[1,2,5,8,10,12],modular:5,monitor:[2,3,5,8,11,15],monospac:2,month:[4,11],monthli:[4,11],more:[1,2,3,5,7,8,11,12,14,15],morn:11,most:[1,2,12,14,15],mostli:[1,5,11],mount:[1,3],move:[1,2,4,5],msg:1,mssql:[1,6],mssql_conn_id:1,mssql_default:1,mssqlhook:1,mssqloper:1,mssqltohivetr
 ansf:1,much:[1,2,11],multi:5,multipart:1,multipart_byt:1,multipl:[1,2],multipli:1,multiprocess:1,must:[1,2,3,4,7,12,14],mutat:2,my_conn_str:3,my_dag1:2,my_dag2:2,my_dag:2,my_databas:1,my_funct:2,my_param:14,my_tabl:1,my_useful_packag:2,myarg:1,mypackag:12,mys3conn:3,mysql:[1,2,3,6],mysql_conn_id:1,mysql_default:1,mysql_postoper:1,mysql_preoper:1,mysql_tabl:1,mysqldb:1,mysqlhook:1,mysqloper:[1,3],mysqltohivetransf:1,name:[1,2,3,7,8,12],namespac:[1,8],nav:12,navig:12,ndfor:14,necessarili:[5,7],need:[1,2,3,4,5,6,8,11,12,13,14],neg:[1,11],neglig:7,neighbour:1,nest:[1,14],network:[1,12],network_mod:1,never:[0,1,4],new_user_email:12,new_user_nam:12,newli:5,newobject:1,next:[1,2,5,11,12],nhere:1,nice:2,night:2,nlst:1,no_confirm:0,no_host_key_check:1,node:[0,1,2],non:[1,3,7],none:[1,2,8,11],noram:12,norandkei:12,normal:[1,2,7],notat:1,note:1,notebook:2,noth:[1,7],notic:[2,7,14],notif:1,notifi:1,notwithstand:7,nov:10,now:[1,2,3,4,12,14],nuanc:2,num:1,num_run:0,num_shard:1,number:[0,1,2,3,5],
 oauth2:12,oauth_callback_rout:12,oauth_key_from_github_enterpris:12,oauth_secret_from_github_enterpris:12,obj:1,object:[0,1,2,3,4,7,8,11,14],objectclass:12,oblig:7,observ:1,obtain:7,occur:[1,2],occurr:1,octet:1,off:[1,3,11],offer:[7,8],offici:[1,10],offset:1,often:[1,2],on_failure_callback:1,on_kil:1,on_retry_callback:1,on_success_callback:1,onboard:4,onc:[1,2,3,4,11,14],one_fail:[1,2],one_success:[1,2],onli:[0,1,2,3,4,6,7,11,12,13],only_fail:0,only_if_upd:1,only_run:0,oozi:5,op1:2,op2:2,op3:2,op4:2,op_arg:1,op_kwarg:1,open:[1,2,4,10,12],oper:0,oppos:[1,2],optim:1,option:[0,1,2],orchestr:[1,5],order:[1,2,3,11,12],org:[1,7,8,10,14],organ:[2,8],organiz:12,origin:7,orm:1,other:[1,2,3,5,7,9,11,14],otherwis:[1,7,14],our:14,out:[1,2],outag:8,outgrow:13,outlier:15,output:[1,14],output_encod:1,output_format:1,outsid:11,outstand:7,over:[0,1,2,3,13,15],overlap:15,overrid:[0,1,14],overridden:[1,4],overview:15,overwhelm:2,overwrit:1,own:[1,2,3,5,7],owner:[1,2,7,12,14],ownership:7,packag:1,packa
 ge1:2,page:[2,4,7,10,15],pai:[1,4],pair:[1,15],panda:1,parallel:[1,2,3,13],param:[0,1,14],paramet:[0,1,2,3,4,9,14],parameter:[2,5,9],parent:2,parent_dag:[1,2],parent_dag_nam:2,pars:[1,4,8,14],part:[1,2,7,9,12],parti:7,particular:7,partit:[1,2],partition_nam:1,partition_spec:1,partli:4,pass:[1,2,4,8,14],pass_valu:1,passwd:15,password:[1,2,3,4,5,6],password_auth:12,passwordus:12,past:[1,2,4,11,15],patent:7,path2:1,path:[1,2,3],patrick:10,patrickleotardif:10,pattern:[1,2],paus:[0,1],payload:1,pem:1,peopl:[1,8,14],pep:1,percent:7,perfect:2,perform:[0,1,2,3,5,7,15],perhap:[2,15],period:[1,2,4,11,14],permiss:[7,12],perpetu:7,persist:[1,11],perspect:2,pertain:7,phase:8,physic:1,pick:[2,3,9],pickl:[0,1,2,3],picklabl:1,pickle_id:1,pickleabl:1,pid:[0,13],pig:[1,2],pigcmd:1,piggi:8,pigoper:1,pin_100:1,pip:[2,6,13],pipelin:[1,2,3,5,9],pitfal:10,place:[1,2,7,14],placehold:1,plain:[1,3],plan:4,platform:[1,2,5,6],pleas:[2,3,10,12,14],plug:8,plugin:[1,5],pluginexecutor:8,pluginhook:8,pluginoper:8,p
 lugins_manag:8,png:1,point:[1,2,3,4,13,14],poke:1,poke_interv:1,polymorph:1,pool:[0,1],pool_ful:1,popen:1,popul:14,port:[0,1,3,12,13],posit:[0,1],possibl:[0,1,2,3,7,12,13,14,15],post:[1,10],post_execut:1,postgr:[1,2,3,6],postgres_conn_id:1,postgres_default:1,postgres_mast:2,postgreshook:1,postgresoper:1,potenti:2,power:[2,5,7,14],practic:[1,10],pre:4,pre_execut:1,preced:[1,3,14],prefer:[7,11,13],prefix:[1,2,3],preoper:1,prepar:[2,7],prepare_templ:1,prepped_request:1,present:1,preset:11,presto:1,presto_conn_id:1,presto_default:1,prestocheckoper:1,prestocmd:1,prestohook:1,prestointervalcheckoper:1,prestotomysqloper:2,prestovaluecheckoper:1,pretti:14,prevent:[1,2,12],previou:[0,1,2,4,14],previous:2,primari:1,primit:1,princip:[0,2,12],print:[0,1,2,7,14],print_dat:14,print_head:1,prior:1,priorit:2,prioriti:[1,2],priority_weight:[1,2,14],privaci:7,probabl:[3,4,12],problem:2,problemat:2,proce:[1,4],process:[0,1,2,3,5,11,14],process_fil:1,produc:1,product:[1,2,3,5,7,9,11,14],profil:[1,5],pr
 ogram:[1,10],programmat:5,progress:[1,5,14],project:5,promin:7,prompt:0,propag:[1,2],propaget:1,proper:14,properli:[3,4],properti:[1,2,4,12],provid:[1,2,3,7,9,14,15],provide_context:[1,2],proxy_us:[1,12],proxyus:12,psycopg2:3,ptarget:1,publicli:7,publish:1,pull:[1,2],pull_funct:2,puppet:[3,10],pure:2,purpl:1,purpos:[7,14],push:[1,2,8],push_funct:2,pushing_task:2,put:[1,2,8],pwd:2,pydata:1,pyhiv:1,pypi:[6,13],python:[1,2,3,5,6,8,12,14],python_1:2,python_cal:[1,2],python_oper:1,pythonhost:8,pythonoper:[1,2],pythonpath:[3,12],qualif:12,qualifi:12,qualiti:[1,2],qubol:[1,6],qubole_conn_id:1,qubole_default:1,quboleoper:1,queri:[1,3,5],queu:[1,2,4],queue:[0,1],quick:[3,5],quickli:[9,13,14,15],quit:4,r39132:10,rabbitmq:[3,6],rais:[1,14],ran:0,randkei:12,random:1,randomli:2,rang:[0,1,11,14],rather:2,ratio:1,raw:1,reach:[1,2,3,4],reactiv:4,read:[1,3,4,12,14],readabl:[1,7,12],readi:[1,5,13],readm:3,readthedoc:14,ready_for_retri:1,real:3,realli:[1,12,14],realm:12,reason:[4,7,14],reboot:2,rebuil
 d:0,recap:5,receiv:[1,2,7,11],recent:[1,2],recip:10,recipi:7,recogn:1,recommend:[1,3,4,7],record:[0,1,2,14],recreat:1,recurs:1,red:1,redhat:3,redi:3,redirect:0,redistribut:7,redund:14,referenc:[2,14],reflect:[2,5,8,14],refrain:2,refresh:1,refresh_from_db:1,regard:[1,7],regardless:[0,1],regex:[0,1],regex_kei:1,regexp:0,regist:[8,9,12],regular:[1,11],reinit_frequ:12,rel:[1,11,14],relat:[1,2,3,4,6],relationship:2,relativedelta:1,relev:4,reli:[1,12],reliabl:1,remain:[7,12],rememb:[1,14],remot:[1,3,11],remote_base_log_fold:3,remote_full_path:1,remote_host:1,remote_log_conn_id:3,remote_port:1,remov:1,render:[0,1,2,8,14],render_templ:1,render_template_from_field:1,renew:[0,12],repeat:2,repl:12,replac:[1,7,12],report:[0,1],repositori:[3,14],repres:[1,2,4,7,11],represent:[1,15],reproduc:7,reproduct:7,request:[0,1,2],requir:[1,2,3,5,6,7,12],resembl:2,reset:0,resetdb:0,resid:2,resolv:1,resourc:[0,1,2,5],respect:[0,1,2,14],respons:[1,7],response_check:1,rest:1,restart:[2,3,4],restrict:12,restru
 cturedtext:2,result:[1,2,7,9,14],resum:0,retain:7,retri:[1,2,14],retriev:[1,2],retrieve_fil:1,retry_delai:[1,14],return_valu:1,reus:8,review:5,revis:7,riccomini:[5,10],rich:[0,2,5,15],rid:1,right:[1,2,7,9],rightmost:2,risk:7,roadmap:5,role:3,roll:5,room:1,room_id:1,root:[1,2],root_dag:1,round:[1,4],rout:12,routin:[1,2],row:[1,2],royalti:7,rule:1,run:[0,1,2,3,4,5],run_a:1,run_and_check:1,run_as_own:[1,12],run_cli:1,run_id:[0,1,11],runme_0:13,runnabl:[1,2],rwc:12,s3_conn_id:1,s3_default:1,s3_kei:1,s3connect:1,s3distcp:1,s3filetransferoper:2,s3hook:[1,3],s3keysensor:[1,6],s3prefixsensor:6,s3tohivetransf:1,safe_mod:1,sai:[1,2,4,11],said:2,samaccountnam:12,samba:[1,6],samba_conn_id:1,samba_default:1,same:[1,2,3,7,9,12],sampl:[1,3],sample_s:1,sandbox:3,save:[3,4,9,14],scala:1,scalabl:5,scan:[1,2],scene:[6,11],schedul:[0,1,2,3,4,5],schedule_interv:[1,2,4,11,14],scheduler_interv:4,scheduler_run:3,schema:[1,2,3],scratch:14,screen:14,screenshot:5,script:[1,2,3,4,5],script_begin_tag:1,script_l
 oc:1,search:[1,2],search_path:3,search_scop:12,second:[1,2,4,14],secret:[12,15],section:[1,2,3,4,7,12,13,14],secur:[1,2,5,7],see:[1,2,3,4,7,11,12,13,14,15],seen:[3,15],segment:1,select:[1,2,9,11,12],self:[1,8],sell:7,semicolon:1,send:[0,1,2,10],send_room_notif:1,sender:1,sensor:[1,2,4],sensor_queu:2,sent:[1,2,3,7],separ:[1,2,7,14],sequenti:[1,11,13],sequentialexecutor:[1,2,13],serial:[0,1],serv:[0,3,14],serve_log:[0,3],server:[0,1,3,8,12,13,14],servic:[0,1,2,3,6,7,11,12],session:[1,12],set:[0,1,2],set_depend:[1,14],set_downstream:[1,2,14],set_the_password:12,set_upstream:[1,2,14],setgid:3,setuid:3,setup:[3,12],sever:2,shall:7,shape:11,share:[1,2,3,7,8,9],shell:12,shellcmd:1,ship:[0,1],ship_dag:0,shop:7,shortcircuitoper:1,shortcut:[3,15],shortest:1,should:[1,2,3,4,7,8,11,13,14],shouldn:[1,2],show:[0,1,2,4,8,15],shown:1,sid:10,siddharth:10,side:[1,3],signal:1,similar:[2,3,5],simpl:[1,2,8,9,14],simplehttpoper:1,simpler:14,simplest:12,simpli:[1,2,8,11,12,14],simplifi:1,simul:14,simultan
 :2,sinc:[1,3,13,14],sing:1,singl:[0,1,2,14],site:12,size:[1,2],skew:8,skip:[0,1,2],sla:1,sla_miss_callback:1,slack:[1,6],slack_oper:1,slackapioper:1,slackapipostoper:[1,6],slackoper:2,slave:3,sleep:14,slightli:5,slot:[1,2],slowli:5,slug:12,small:[1,2],smaller:1,smallish:1,snap:5,sock:1,soft_fail:1,softwar:[5,7,10],sole:7,solut:5,some:[1,2,3,4,11,14,15],somehow:3,someth:[2,12,14],sometim:[1,2,14],somewhat:14,somewher:13,soon:[1,2,3,4,11],sort:2,sound:[2,14],sourc:[1,2,7,9,10,15],source_conn_id:1,source_project_dataset_t:1,space:5,span:15,spark:[1,2,5],sparkcmd:1,spawn:3,special:[1,2,4,7],specif:[0,1,2,3,4,7,11,14,15],specifi:[0,1,2,3,4,5,11,12,14],specific_valu:1,spent:15,split:1,split_column:1,sponsor:5,spring:10,sql:[1,2,3,6,9],sql_alchemy_conn:3,sql_alchemy_conn_cmd:3,sqlachemi:1,sqlalchemi:[1,2,3],sqlite:[1,13],sqlitehook:1,sqloper:2,sqlsensor:1,squeez:1,sscursor:1,ssh:[1,12],ssh_default:1,ssh_hook:1,sshexecuteoper:1,sshhook:1,sshpass:1,ssl:1,sslcert:1,sslmode:1,stabil:5,stabl:6,
 stack:[6,8],stage:[1,2],stai:11,stamp:11,stand:2,standalon:2,standard:[1,2],stanza:3,start:[0,1,2,3,4,5,6,10,11],start_dat:[0,1,2,4,11,14],stat:1,state:[0,1,2,4,7,11,14],state_d:1,statement:[1,2,7],static_babynam:1,static_babynames_partit:1,static_fold:8,static_path:1,static_url_path:8,statu:[0,1,2,3,4,5,7,11,13,14,15],stderr:0,stdout:[0,1,14],step:[1,3,14,15],steven:10,still:[1,4],sting:1,stop:[1,3],stoppag:7,storag:[1,3],store:[0,1,2,3,8,12,13],store_fil:1,storm:5,str:[1,8,11],straightforward:13,stream:[1,5],string:[0,1,2,3,14],string_data:1,structur:[1,5,14],style:1,sub:[0,1],sub_command:1,sub_commnad:1,sub_dag:[1,2],sub_part:1,subcommand:[1,3,11,14],subdag:[0,1],subdagoper:2,subdir:0,subdirectori:2,subject:[1,7],sublicens:7,submiss:[7,11],submit:[1,7],subpackag:[6,12],subpartit:1,subprocess:[1,11],subscrib:[2,10],subsect:[0,11],subsequ:[0,7,11],subset:1,subtl:2,subtract:1,subtre:12,succe:[1,2],succeed:[0,1,2,4,15],success:[0,1,2,4,5,11,14],successfulli:[1,2,14],suit:5,sum:2,summ
 it:10,sundai:11,super_us:12,supersed:7,superus:12,superuser_filt:12,suppli:[1,3,12],support:[0,1,3,6,7,12],sure:[1,2,3,4,12,14],surfac:14,surgeri:5,sync:[0,1,3,11],sync_to_db:1,synchron:[1,3],syntax:7,sysconfig:3,system:[1,2,3,7],syvineckruyk:10,tab:[1,12],tabl:[1,2,8],table_exist:1,table_nam:[1,2],tablenam:1,tag:[1,4],take:[1,2,3,10,12,13,14,15],talk:[1,10,14],tardif:10,target:[1,3,11],target_field:1,target_partition_s:1,target_tim:1,task:[0,1],task_id:[0,1,2,11,14],task_inst:[1,2],task_instance_key_str:1,task_or_task_list:1,task_param:0,task_regex:[0,1],task_stat:0,taskinst:[1,2],tbl:1,team:[1,12],tell:[0,1],templat:[0,1],template_fold:8,template_searchpath:[1,14],templated_command:14,templates_dict:1,templates_ext:1,temporari:[1,2],tenanc:5,tenant:12,term:7,termin:[2,7],test:[0,1,2,3,4,5,8,12],test_env:2,test_hql:1,test_mod:1,test_plugin:8,testabl:5,testview:8,text:[1,3,7,15],textfil:1,than:[1,2,5,14],thei:[0,1,2,3,5,6,8,9,11,14],them:[0,1,2,8,11,14],themselv:2,theoret:1,theori:[
 4,7],thereof:7,thi:[0,1,2,3,4,5,7,8,11,12,13,14,15],thin:1,thing:[1,2,4,14],think:[5,14],third:7,this_dag_will_be_discov:2,those:[1,2,3,7,12],though:[2,5],thousand:2,thread:1,three:[2,3],threshold:8,thrift:1,through:[1,2,6,7,11,13,14],throughout:1,ticket:[0,12],tighten:12,time:[1,2,3,4,8,11,12,14,15],timedelta:[1,2,4,11,14],timedeltasensor:[1,4],timefram:1,timeout:[0,1,2],timesensor:1,timestamp:[1,2,11],tip:[2,10],titl:[2,7],tls_ca_cert:1,tls_client_cert:1,tls_client_kei:1,tls_hostnam:1,tls_ssl_version:1,tmp:[1,2],tmp_dir:1,tmp_file:1,todai:1,togeth:[1,2],token:1,toler:[1,2],tomorrow:1,tomorrow_d:1,tomorrow_ds_nodash:1,too:[1,2,11],tool:[1,2,6,8],toolbox:8,top:[1,3,5],topic:3,tornado:0,tort:7,tour:[13,14],track:[2,7,14],tracker:1,trade:7,trademark:7,tradit:1,tradition:2,train:7,transact:1,transfer:[1,2,7],transform:7,translat:[1,7],transpar:15,travers:1,treat:[1,2],tree:[0,1,4,5,14],tree_view:1,tri:[1,12],trick:10,trigger:[0,1],trigger_dag:[0,11],trigger_dag_id:1,trigger_rul:[1,2],t
 riggerdagrunoper:[1,10],triggerrul:1,troubl:6,troubleshoot:[5,15],truncat:1,truth:1,ts_dim:1,ts_nodash:1,tsv:1,tty:1,tunnel:[1,12],tupl:1,turn:[2,12],tutori:[5,13],twice:1,two:[1,2,14],txt:1,type:[0,1,2,4,7,9,12],typic:[1,2],ubuntu:3,udf_config:1,uid:12,ultim:[1,14],unarchiv:1,unassign:2,unavail:3,unblock:11,undefin:12,under:[1,2,3,7,8,10],undergo:5,underli:8,underscor:3,understand:[1,4,8,14,15],undesir:11,unexpect:2,unexpectedli:2,union:7,uniqu:[1,14],unit:[3,5,12],unittest:1,unix:1,unless:[4,7],unpack:[1,2],unpaus:0,unset:1,unsound:[1,2],unsuccess:1,unsupport:1,until:[1,2,3,5],up_for_retri:1,updat:[1,2,4,12],updateonli:1,upgrad:0,upgradedb:0,upload:1,upon:[2,3,13],upstream:[0,1,2,4,11],upstream_fail:[1,2],upstream_list:1,upstream_task_id:1,uri:[1,2,3,12],url:[1,2,3,8,9,12],usag:[0,14],use_beelin:[1,12],user:[0,1,2,3,5,6,12,14],user_defined_macro:1,user_filt:12,user_name_attr:12,user_program_argu:1,usernam:[1,3,12],usr:3,usual:[2,3],utf8:1,utf:1,util:[1,5,13],uuid:1,val:[0,1],valid
 :[4,5,12],valu:[0,1,2,3,4,12,14,15],valuecheckoper:1,variabl:0,vast:1,verbal:7,verbos:[1,14],veri:[0,1,2,4,10,13,14],verifi:[2,4],version:[0,1,2,5,6,7],vertia:1,vertica:[1,6],vertica_conn_id:1,vertica_default:1,verticahook:1,verticaoper:1,verticaql:1,verticatohivetransf:1,via:[1,2,12],video:[1,10],view:[0,1,2,3,4,5,8],virtual:1,virtualenv:2,visibl:[1,2],visual:[5,8,9,14,15],vocabulari:2,volum:1,wai:[1,2,3,4,6,8,11,12,14,15],wait:[0,1,2],wait_for_downstream:[1,4],wait_for_empty_queu:2,walk:14,want:[1,2,3,4,8,11,12,14],warranti:7,wasn:4,wasnt:1,watch:[1,3,4],weapon:9,web:[0,1,2,3,5,8],webhdfs_conn_id:1,webhdfs_default:1,webhdfshook:1,webhdfssensor:1,webserv:[0,3,4,12,13,14],week:11,weekli:11,weight:1,well:[1,2,3,4,11],were:1,what:[1,2,4,5,6],whatev:[2,3,6],when:[1,2,3,4,5,8,9,11,12,14],whenev:1,where:[1,2,3,4,7,11,12,14,15],where_claus:1,wherev:[1,7],whether:[1,7,11],which:[0,1,2,3,4,7,11,12,13,14],whitelist:12,who:8,whole:[1,2,7],whom:7,whose:11,why:[4,5],wiedmer:10,wiki:10,wildcard:
 1,wildcard_kei:1,wildcard_match:1,win:1,window:11,winter:10,wire:2,within:[0,1,2,3,4,7],without:[0,1,2,7,11,12],won:[1,4,6],word:[2,3,11,15],work:[0,1,2,3,4,5,7,8,9,11,12,13],worker:[0,1,2,3,4,5,6,8,12,14],worker_timeout:0,workerclass:0,workflow:1,workload:8,worldwid:7,worri:14,worth:1,would:[1,2,3,4,5,11,12,14],wouldn:[1,4],wrap:14,wrapper:1,write:[3,5,7,8,9,14],write_disposit:1,write_empti:1,written:[1,7,14],wrong:14,www:[1,2,7],x01:1,xcom:1,xcom_al:1,xcom_pul:[1,2],xcom_push:[1,2],xcom_return_kei:1,xml:12,xst:12,yaml:2,year:[1,11],yearli:11,yellow:1,yesterdai:1,yesterday_d:1,yesterday_ds_nodash:1,yet:[1,2,5,12,15],you:[1,2,3,4,5,6,7,8,9,11,12,13,14,15],your:[1,2,3,4,5,6,7,8,9,11],yourself:1,youtub:1,yum:6,yvinec:10,yyyi:[0,1,2],yyyymmdd:1,zero:1,zip:[1,2],zip_dag:2,zip_dag_cont:2,zoom:2},titles:["Command Line Interface","API Reference","Concepts","Configuration","FAQ","Apache Airflow (incubating) Documentation","Installation","License","Plugins","Data Profiling","Project","Schedu
 ling & Triggers","Security","Quick Start","Tutorial","UI / Screenshots"],titleterms:{"default":[1,2,14],"function":2,"import":14,addit:2,adhoc:9,airflow:[5,6,8],apach:5,api:1,argument:[2,14],assign:2,authent:12,backend:3,backfil:14,baseoper:1,basesensoroper:1,beyond:5,bitshift:2,branch:2,build:8,celeri:3,chart:[9,15],cluster:2,code:15,command:[0,14],committ:10,commun:[1,3],composit:2,concept:2,configur:3,connect:[2,3],content:5,context:[2,15],contribut:[1,3],core:2,dag:[2,11,14,15],data:9,definit:14,depend:14,document:[2,5],durat:15,enabl:12,enterpris:12,exampl:[8,14],executor:1,extern:11,extra:6,faq:4,file:14,form:9,gantt:15,get:6,ghe:12,github:12,graph:15,histori:10,hook:[1,2],horizon:5,idea:2,incub:5,instal:6,instanc:[2,15],instanti:14,integr:3,interfac:[0,8],jinja:[2,14],keep:11,kerbero:12,ldap:12,licens:7,limit:12,line:[0,14],link:10,log:3,macro:1,manag:2,menu:15,meso:3,metadata:14,mind:11,model:1,modul:14,multi:12,next:[13,14],note:2,oper:[1,2],option:3,out:3,own:12,packag
 :[2,6],password:12,pipelin:14,plugin:8,polici:2,pool:2,principl:5,profil:9,project:10,queri:9,queue:2,quick:13,recap:14,refer:1,resourc:10,roadmap:10,roll:12,rule:2,run:[11,14],scale:3,schedul:11,scope:2,screenshot:[9,15],script:14,secur:12,set:[3,12,14],sla:2,start:13,subdag:2,systemd:3,task:[2,14,15],templat:[2,14],tenanc:12,test:14,top:8,tree:15,trigger:[2,11],tutori:14,undead:2,upstart:3,valid:14,variabl:[1,2,15],view:15,web:12,what:[8,13,14],why:8,workflow:2,xcom:2,your:12,zombi:2}})
\ No newline at end of file

[5/6] incubator-airflow-site git commit: Adding incubator related disclaimer

[ma...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/502934ca/_modules/airflow/contrib/operators/bigquery_to_gcs.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/bigquery_to_gcs.html b/_modules/airflow/contrib/operators/bigquery_to_gcs.html
new file mode 100644
index 0000000..465128c
--- /dev/null
+++ b/_modules/airflow/contrib/operators/bigquery_to_gcs.html
@@ -0,0 +1,278 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>airflow.contrib.operators.bigquery_to_gcs &mdash; Airflow Documentation</title>
+  
+
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+    <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+        <link rel="up" title="Module code" href="../../../index.html"/> 
+
+  
+  <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+                <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+        <a href="../../../../index.html">Airflow</a>
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+  <ul class="wy-breadcrumbs">
+    <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+      
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+      
+    <li>airflow.contrib.operators.bigquery_to_gcs</li>
+      <li class="wy-breadcrumbs-aside">
+        
+          
+        
+      </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for airflow.contrib.operators.bigquery_to_gcs</h1><div class="highlight"><pre>
+<span></span><span class="kn">import</span> <span class="nn">logging</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.bigquery_hook</span> <span class="kn">import</span> <span class="n">BigQueryHook</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
+
+<div class="viewcode-block" id="BigQueryToCloudStorageOperator"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.bigquery_to_gcs.BigQueryToCloudStorageOperator">[docs]</a><span class="k">class</span> <span class="nc">BigQueryToCloudStorageOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Transfers a BigQuery table to a Google Cloud Storage bucket.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;source_project_dataset_table&#39;</span><span class="p">,</span><span class="s1">&#39;destination_cloud_storage_uris&#39;</span><span class="p">,)</span>
+    <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;.sql&#39;</span><span class="p">,)</span>
+    <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#e4e6f0&#39;</span>
+
+    <span class="nd">@apply_defaults</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">source_project_dataset_table</span><span class="p">,</span>
+        <span class="n">destination_cloud_storage_uris</span><span class="p">,</span>
+        <span class="n">compression</span><span class="o">=</span><span class="s1">&#39;NONE&#39;</span><span class="p">,</span>
+        <span class="n">export_format</span><span class="o">=</span><span class="s1">&#39;CSV&#39;</span><span class="p">,</span>
+        <span class="n">field_delimiter</span><span class="o">=</span><span class="s1">&#39;,&#39;</span><span class="p">,</span>
+        <span class="n">print_header</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span>
+        <span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">&#39;bigquery_default&#39;</span><span class="p">,</span>
+        <span class="n">delegate_to</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+        <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+        <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Create a new BigQueryToCloudStorage to move data from BigQuery to</span>
+<span class="sd">        Google Cloud Storage.  See here:</span>
+
+<span class="sd">        https://cloud.google.com/bigquery/docs/reference/v2/jobs</span>
+
+<span class="sd">        For more details about these parameters.</span>
+
+<span class="sd">        :param source_project_dataset_table: The dotted (&lt;project&gt;.)&lt;dataset&gt;.&lt;table&gt; BigQuery table to use as the</span>
+<span class="sd">            source data. If &lt;project&gt; is not included, project will be the project defined in the connection json.</span>
+<span class="sd">        :type source_project_dataset_table: string</span>
+<span class="sd">        :param destination_cloud_storage_uris: The destination Google Cloud</span>
+<span class="sd">            Storage URI (e.g. gs://some-bucket/some-file.txt). Follows</span>
+<span class="sd">            convention defined here:</span>
+<span class="sd">            https://cloud.google.com/bigquery/exporting-data-from-bigquery#exportingmultiple</span>
+<span class="sd">        :type destination_cloud_storage_uris: list</span>
+<span class="sd">        :param compression: Type of compression to use.</span>
+<span class="sd">        :type compression: string</span>
+<span class="sd">        :param export_format: File format to export.</span>
+<span class="sd">        :type field_delimiter: string</span>
+<span class="sd">        :param field_delimiter: The delimiter to use when extracting to a CSV.</span>
+<span class="sd">        :type field_delimiter: string</span>
+<span class="sd">        :param print_header: Whether to print a header for a CSV file extract.</span>
+<span class="sd">        :type print_header: boolean</span>
+<span class="sd">        :param bigquery_conn_id: reference to a specific BigQuery hook.</span>
+<span class="sd">        :type bigquery_conn_id: string</span>
+<span class="sd">        :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd">            For this to work, the service account making the request must have domain-wide delegation enabled.</span>
+<span class="sd">        :type delegate_to: string</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">BigQueryToCloudStorageOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">source_project_dataset_table</span> <span class="o">=</span> <span class="n">source_project_dataset_table</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">destination_cloud_storage_uris</span> <span class="o">=</span> <span class="n">destination_cloud_storage_uris</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">compression</span> <span class="o">=</span> <span class="n">compression</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">export_format</span> <span class="o">=</span> <span class="n">export_format</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">field_delimiter</span> <span class="o">=</span> <span class="n">field_delimiter</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">print_header</span> <span class="o">=</span> <span class="n">print_header</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span> <span class="o">=</span> <span class="n">bigquery_conn_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+
+    <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Executing extract of </span><span class="si">%s</span><span class="s1"> into: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">source_project_dataset_table</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">destination_cloud_storage_uris</span><span class="p">)</span>
+        <span class="n">hook</span> <span class="o">=</span> <span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span><span class="p">,</span> <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+        <span class="n">conn</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+        <span class="n">cursor</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span>
+        <span class="n">cursor</span><span class="o">.</span><span class="n">run_extract</span><span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">source_project_dataset_table</span><span class="p">,</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">destination_cloud_storage_uris</span><span class="p">,</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">compression</span><span class="p">,</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">export_format</span><span class="p">,</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">field_delimiter</span><span class="p">,</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">print_header</span><span class="p">)</span></div>
+</pre></div>
+
+           </div>
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'../../../../',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true
+        };
+    </script>
+      <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+      <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+      <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/502934ca/_modules/airflow/contrib/operators/gcs_download_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/gcs_download_operator.html b/_modules/airflow/contrib/operators/gcs_download_operator.html
new file mode 100644
index 0000000..4af6020
--- /dev/null
+++ b/_modules/airflow/contrib/operators/gcs_download_operator.html
@@ -0,0 +1,253 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>airflow.contrib.operators.gcs_download_operator &mdash; Airflow Documentation</title>
+  
+
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+    <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+        <link rel="up" title="Module code" href="../../../index.html"/> 
+
+  
+  <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+                <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+        <a href="../../../../index.html">Airflow</a>
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+  <ul class="wy-breadcrumbs">
+    <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+      
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+      
+    <li>airflow.contrib.operators.gcs_download_operator</li>
+      <li class="wy-breadcrumbs-aside">
+        
+          
+        
+      </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for airflow.contrib.operators.gcs_download_operator</h1><div class="highlight"><pre>
+<span></span><span class="kn">import</span> <span class="nn">logging</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcs_hook</span> <span class="kn">import</span> <span class="n">GoogleCloudStorageHook</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
+
+<div class="viewcode-block" id="GoogleCloudStorageDownloadOperator"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.gcs_download_operator.GoogleCloudStorageDownloadOperator">[docs]</a><span class="k">class</span> <span class="nc">GoogleCloudStorageDownloadOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Downloads a file from Google Cloud Storage.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;bucket&#39;</span><span class="p">,</span><span class="s1">&#39;object&#39;</span><span class="p">,</span><span class="s1">&#39;filename&#39;</span><span class="p">,)</span>
+    <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;.sql&#39;</span><span class="p">,)</span>
+    <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#f0eee4&#39;</span>
+
+    <span class="nd">@apply_defaults</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">bucket</span><span class="p">,</span>
+        <span class="nb">object</span><span class="p">,</span>
+        <span class="n">filename</span><span class="p">,</span>
+        <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="s1">&#39;google_cloud_storage_default&#39;</span><span class="p">,</span>
+        <span class="n">delegate_to</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+        <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+        <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Create a new GoogleCloudStorageDownloadOperator.</span>
+
+<span class="sd">        :param bucket: The Google cloud storage bucket where the object is.</span>
+<span class="sd">        :type bucket: string</span>
+<span class="sd">        :param object: The name of the object to download in the Google cloud</span>
+<span class="sd">            storage bucket.</span>
+<span class="sd">        :type object: string</span>
+<span class="sd">        :param filename: The file path on the local file system (where the</span>
+<span class="sd">            operator is being executed) that the file should be downloaded to.</span>
+<span class="sd">        :type filename: string</span>
+<span class="sd">        :param google_cloud_storage_conn_id: The connection ID to use when</span>
+<span class="sd">            connecting to Google cloud storage.</span>
+<span class="sd">        :type google_cloud_storage_conn_id: string</span>
+<span class="sd">        :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd">            For this to work, the service account making the request must have domain-wide delegation enabled.</span>
+<span class="sd">        :type delegate_to: string</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">GoogleCloudStorageDownloadOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span> <span class="o">=</span> <span class="n">bucket</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">object</span> <span class="o">=</span> <span class="nb">object</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">filename</span> <span class="o">=</span> <span class="n">filename</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span> <span class="o">=</span> <span class="n">google_cloud_storage_conn_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+
+    <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Executing download: </span><span class="si">%s</span><span class="s1">, </span><span class="si">%s</span><span class="s1">, </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">object</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">filename</span><span class="p">)</span>
+        <span class="n">hook</span> <span class="o">=</span> <span class="n">GoogleCloudStorageHook</span><span class="p">(</span><span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span><span class="p">,</span>
+                                      <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+        <span class="k">print</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">download</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">object</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">filename</span><span class="p">))</span></div>
+</pre></div>
+
+           </div>
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'../../../../',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true
+        };
+    </script>
+      <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+      <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+      <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/502934ca/_modules/airflow/models.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/models.html b/_modules/airflow/models.html
index d382539..aba90cd 100644
--- a/_modules/airflow/models.html
+++ b/_modules/airflow/models.html
@@ -965,8 +965,8 @@
 <span class="sd">        Refreshes the task instance from the database based on the primary key</span>
 
 <span class="sd">        :param lock_for_update: if True, indicates that the database should</span>
-<span class="sd">        lock the TaskInstance (issuing a FOR UPDATE clause) until the session</span>
-<span class="sd">        is committed.</span>
+<span class="sd">            lock the TaskInstance (issuing a FOR UPDATE clause) until the</span>
+<span class="sd">            session is committed.</span>
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="n">TI</span> <span class="o">=</span> <span class="n">TaskInstance</span>
 
@@ -1325,7 +1325,6 @@
         <span class="bp">self</span><span class="o">.</span><span class="n">test_mode</span> <span class="o">=</span> <span class="n">test_mode</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">force</span> <span class="o">=</span> <span class="n">force</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">refresh_from_db</span><span class="p">(</span><span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">,</span> <span class="n">lock_for_update</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">clear_xcom_data</span><span class="p">()</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">job_id</span> <span class="o">=</span> <span class="n">job_id</span>
         <span class="n">iso</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">hostname</span> <span class="o">=</span> <span class="n">socket</span><span class="o">.</span><span class="n">getfqdn</span><span class="p">()</span>
@@ -1334,6 +1333,7 @@
         <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">State</span><span class="o">.</span><span class="n">RUNNING</span><span class="p">:</span>
             <span class="n">logging</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">&quot;Another instance is running, skipping.&quot;</span><span class="p">)</span>
         <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">State</span><span class="o">.</span><span class="n">REMOVED</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">clear_xcom_data</span><span class="p">()</span>
             <span class="n">logging</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;Task {} was removed from the dag&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="p">))</span>
         <span class="k">elif</span> <span class="ow">not</span> <span class="n">force</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">State</span><span class="o">.</span><span class="n">SUCCESS</span><span class="p">:</span>
             <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
@@ -1358,6 +1358,7 @@
                 <span class="s2">&quot;Next run after {0}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">next_run</span><span class="p">)</span>
             <span class="p">)</span>
         <span class="k">elif</span> <span class="n">force</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="ow">in</span> <span class="n">State</span><span class="o">.</span><span class="n">runnable</span><span class="p">():</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">clear_xcom_data</span><span class="p">()</span>
             <span class="n">HR</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="se">\n</span><span class="s2">&quot;</span> <span class="o">+</span> <span class="p">(</span><span class="s2">&quot;-&quot;</span> <span class="o">*</span> <span class="mi">80</span><span class="p">)</span> <span class="o">+</span> <span class="s2">&quot;</span><span class="se">\n</span><span class="s2">&quot;</span>  <span class="c1"># Line break</span>
 
             <span class="c1"># For reporting purposes, we report based on 1-indexed,</span>
@@ -3583,7 +3584,11 @@
                      <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">tis</span><span class="p">)))</span>
 
         <span class="k">for</span> <span class="n">ti</span> <span class="ow">in</span> <span class="n">tis</span><span class="p">:</span>
-            <span class="n">ti</span><span class="o">.</span><span class="n">task</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">get_task</span><span class="p">(</span><span class="n">ti</span><span class="o">.</span><span class="n">task_id</span><span class="p">)</span>
+            <span class="c1"># skip in db?</span>
+            <span class="k">if</span> <span class="n">ti</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">State</span><span class="o">.</span><span class="n">REMOVED</span><span class="p">:</span>
+                <span class="n">tis</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="n">ti</span><span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">ti</span><span class="o">.</span><span class="n">task</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">get_task</span><span class="p">(</span><span class="n">ti</span><span class="o">.</span><span class="n">task_id</span><span class="p">)</span>
 
         <span class="c1"># pre-calculate</span>
         <span class="c1"># db is faster</span>
@@ -3650,8 +3655,11 @@
         <span class="n">task_ids</span> <span class="o">=</span> <span class="p">[]</span>
         <span class="k">for</span> <span class="n">ti</span> <span class="ow">in</span> <span class="n">tis</span><span class="p">:</span>
             <span class="n">task_ids</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">ti</span><span class="o">.</span><span class="n">task_id</span><span class="p">)</span>
-            <span class="k">if</span> <span class="ow">not</span> <span class="n">dag</span><span class="o">.</span><span class="n">get_task</span><span class="p">(</span><span class="n">ti</span><span class="o">.</span><span class="n">task_id</span><span class="p">)</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">State</span><span class="o">.</span><span class="n">unfinished</span><span class="p">():</span>
-                <span class="n">ti</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="n">State</span><span class="o">.</span><span class="n">REMOVED</span>
+            <span class="k">try</span><span class="p">:</span>
+                <span class="n">dag</span><span class="o">.</span><span class="n">get_task</span><span class="p">(</span><span class="n">ti</span><span class="o">.</span><span class="n">task_id</span><span class="p">)</span>
+            <span class="k">except</span> <span class="n">AirflowException</span><span class="p">:</span>
+                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="ow">is</span> <span class="ow">not</span> <span class="n">State</span><span class="o">.</span><span class="n">RUNNING</span><span class="p">:</span>
+                    <span class="n">ti</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="n">State</span><span class="o">.</span><span class="n">REMOVED</span>
 
         <span class="c1"># check for missing tasks</span>
         <span class="k">for</span> <span class="n">task</span> <span class="ow">in</span> <span class="n">dag</span><span class="o">.</span><span class="n">tasks</span><span class="p">:</span>

[2/6] incubator-airflow-site git commit: Adding incubator related disclaimer

[ma...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/502934ca/genindex.html
----------------------------------------------------------------------
diff --git a/genindex.html b/genindex.html
index 05f8c42..ecba6b9 100644
--- a/genindex.html
+++ b/genindex.html
@@ -152,6 +152,7 @@
  | <a href="#M"><strong>M</strong></a>
  | <a href="#O"><strong>O</strong></a>
  | <a href="#P"><strong>P</strong></a>
+ | <a href="#Q"><strong>Q</strong></a>
  | <a href="#R"><strong>R</strong></a>
  | <a href="#S"><strong>S</strong></a>
  | <a href="#T"><strong>T</strong></a>
@@ -173,10 +174,6 @@
   </dt>
 
       
-  <dt><a href="code.html#module-airflow.contrib.executors">airflow.contrib.executors (module)</a>
-  </dt>
-
-      
   <dt><a href="code.html#module-airflow.contrib.hooks">airflow.contrib.hooks (module)</a>
   </dt>
 
@@ -240,9 +237,21 @@
   <dt><a href="code.html#airflow.operators.BashOperator">BashOperator (class in airflow.operators)</a>
   </dt>
 
+      
+  <dt><a href="code.html#airflow.contrib.hooks.BigQueryHook">BigQueryHook (class in airflow.contrib.hooks)</a>
+  </dt>
+
   </dl></td>
   <td style="width: 33%" valign="top"><dl>
       
+  <dt><a href="code.html#airflow.contrib.operators.bigquery_operator.BigQueryOperator">BigQueryOperator (class in airflow.contrib.operators.bigquery_operator)</a>
+  </dt>
+
+      
+  <dt><a href="code.html#airflow.contrib.operators.bigquery_to_gcs.BigQueryToCloudStorageOperator">BigQueryToCloudStorageOperator (class in airflow.contrib.operators.bigquery_to_gcs)</a>
+  </dt>
+
+      
   <dt><a href="code.html#airflow.operators.BranchPythonOperator">BranchPythonOperator (class in airflow.operators)</a>
   </dt>
 
@@ -402,12 +411,12 @@
   <dt><a href="code.html#airflow.contrib.hooks.FTPHook.delete_file">delete_file() (airflow.contrib.hooks.FTPHook method)</a>
   </dt>
 
-  </dl></td>
-  <td style="width: 33%" valign="top"><dl>
       
   <dt><a href="code.html#airflow.contrib.hooks.FTPHook.describe_directory">describe_directory() (airflow.contrib.hooks.FTPHook method)</a>
   </dt>
 
+  </dl></td>
+  <td style="width: 33%" valign="top"><dl>
       
   <dt><a href="code.html#airflow.models.BaseOperator.detect_downstream_cycle">detect_downstream_cycle() (airflow.models.BaseOperator method)</a>
   </dt>
@@ -417,6 +426,10 @@
   </dt>
 
       
+  <dt><a href="code.html#airflow.contrib.hooks.GoogleCloudStorageHook.download">download() (airflow.contrib.hooks.GoogleCloudStorageHook method)</a>
+  </dt>
+
+      
   <dt><a href="code.html#airflow.models.BaseOperator.downstream_list">downstream_list (airflow.models.BaseOperator attribute)</a>
   </dt>
 
@@ -513,11 +526,19 @@
   </dt>
 
       
-  <dt><a href="code.html#airflow.contrib.hooks.FTPHook.get_conn">get_conn() (airflow.contrib.hooks.FTPHook method)</a>
+  <dt><a href="code.html#airflow.contrib.hooks.BigQueryHook.get_conn">get_conn() (airflow.contrib.hooks.BigQueryHook method)</a>
   </dt>
 
       <dd><dl>
         
+  <dt><a href="code.html#airflow.contrib.hooks.FTPHook.get_conn">(airflow.contrib.hooks.FTPHook method)</a>
+  </dt>
+
+        
+  <dt><a href="code.html#airflow.contrib.hooks.GoogleCloudStorageHook.get_conn">(airflow.contrib.hooks.GoogleCloudStorageHook method)</a>
+  </dt>
+
+        
   <dt><a href="code.html#airflow.contrib.hooks.VerticaHook.get_conn">(airflow.contrib.hooks.VerticaHook method)</a>
   </dt>
 
@@ -592,18 +613,22 @@
   <dt><a href="code.html#airflow.hooks.S3Hook.get_key">get_key() (airflow.hooks.S3Hook method)</a>
   </dt>
 
-  </dl></td>
-  <td style="width: 33%" valign="top"><dl>
       
   <dt><a href="code.html#airflow.hooks.HiveMetastoreHook.get_metastore_client">get_metastore_client() (airflow.hooks.HiveMetastoreHook method)</a>
   </dt>
 
+  </dl></td>
+  <td style="width: 33%" valign="top"><dl>
       
-  <dt><a href="code.html#airflow.hooks.DbApiHook.get_pandas_df">get_pandas_df() (airflow.hooks.DbApiHook method)</a>
+  <dt><a href="code.html#airflow.contrib.hooks.BigQueryHook.get_pandas_df">get_pandas_df() (airflow.contrib.hooks.BigQueryHook method)</a>
   </dt>
 
       <dd><dl>
         
+  <dt><a href="code.html#airflow.hooks.DbApiHook.get_pandas_df">(airflow.hooks.DbApiHook method)</a>
+  </dt>
+
+        
   <dt><a href="code.html#airflow.hooks.HiveServer2Hook.get_pandas_df">(airflow.hooks.HiveServer2Hook method)</a>
   </dt>
 
@@ -631,6 +656,10 @@
 
       </dl></dd>
       
+  <dt><a href="code.html#airflow.contrib.hooks.BigQueryHook.get_service">get_service() (airflow.contrib.hooks.BigQueryHook method)</a>
+  </dt>
+
+      
   <dt><a href="code.html#airflow.hooks.HiveMetastoreHook.get_table">get_table() (airflow.hooks.HiveMetastoreHook method)</a>
   </dt>
 
@@ -650,6 +679,20 @@
   <dt><a href="code.html#airflow.hooks.S3Hook.get_wildcard_key">get_wildcard_key() (airflow.hooks.S3Hook method)</a>
   </dt>
 
+      
+  <dt><a href="code.html#airflow.contrib.operators.gcs_download_operator.GoogleCloudStorageDownloadOperator">GoogleCloudStorageDownloadOperator (class in airflow.contrib.operators.gcs_download_operator)</a>
+  </dt>
+
+      
+  <dt><a href="code.html#airflow.contrib.hooks.GoogleCloudStorageHook">GoogleCloudStorageHook (class in airflow.contrib.hooks)</a>
+  </dt>
+
+      <dd><dl>
+        
+  <dt><a href="code.html#airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook">(class in airflow.contrib.hooks.gcs_hook)</a>
+  </dt>
+
+      </dl></dd>
   </dl></td>
 </tr></table>
 
@@ -721,9 +764,15 @@
 <table style="width: 100%" class="indextable genindextable"><tr>
   <td style="width: 33%" valign="top"><dl>
       
-  <dt><a href="code.html#airflow.hooks.DbApiHook.insert_rows">insert_rows() (airflow.hooks.DbApiHook method)</a>
+  <dt><a href="code.html#airflow.contrib.hooks.BigQueryHook.insert_rows">insert_rows() (airflow.contrib.hooks.BigQueryHook method)</a>
+  </dt>
+
+      <dd><dl>
+        
+  <dt><a href="code.html#airflow.hooks.DbApiHook.insert_rows">(airflow.hooks.DbApiHook method)</a>
   </dt>
 
+      </dl></dd>
       
   <dt><a href="code.html#airflow.macros.integrate_plugins">integrate_plugins() (in module airflow.macros)</a>
   </dt>
@@ -829,6 +878,10 @@
 
       </dl></dd>
       
+  <dt><a href="code.html#airflow.contrib.executors.mesos_executor.MesosExecutor">MesosExecutor (class in airflow.contrib.executors.mesos_executor)</a>
+  </dt>
+
+      
   <dt><a href="code.html#airflow.operators.MetastorePartitionSensor">MetastorePartitionSensor (class in airflow.operators)</a>
   </dt>
 
@@ -931,6 +984,16 @@
   </dl></td>
 </tr></table>
 
+<h2 id="Q">Q</h2>
+<table style="width: 100%" class="indextable genindextable"><tr>
+  <td style="width: 33%" valign="top"><dl>
+      
+  <dt><a href="code.html#airflow.contrib.operators.QuboleOperator">QuboleOperator (class in airflow.contrib.operators)</a>
+  </dt>
+
+  </dl></td>
+</tr></table>
+
 <h2 id="R">R</h2>
 <table style="width: 100%" class="indextable genindextable"><tr>
   <td style="width: 33%" valign="top"><dl>
@@ -1125,6 +1188,12 @@
 <table style="width: 100%" class="indextable genindextable"><tr>
   <td style="width: 33%" valign="top"><dl>
       
+  <dt><a href="code.html#airflow.contrib.hooks.GoogleCloudStorageHook.upload">upload() (airflow.contrib.hooks.GoogleCloudStorageHook method)</a>
+  </dt>
+
+  </dl></td>
+  <td style="width: 33%" valign="top"><dl>
+      
   <dt><a href="code.html#airflow.models.BaseOperator.upstream_list">upstream_list (airflow.models.BaseOperator attribute)</a>
   </dt>
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/502934ca/index.html
----------------------------------------------------------------------
diff --git a/index.html b/index.html
index 7d2271d..383bab0 100644
--- a/index.html
+++ b/index.html
@@ -140,6 +140,18 @@
 <a class="reference internal image-reference" href="_images/incubator.jpg"><img alt="_images/incubator.jpg" src="_images/incubator.jpg" style="width: 150px;" /></a>
 <div class="section" id="apache-airflow-incubating-documentation">
 <h1>Apache Airflow (incubating) Documentation<a class="headerlink" href="#apache-airflow-incubating-documentation" title="Permalink to this headline">�</a></h1>
+<div class="admonition note">
+<p class="first admonition-title">Note</p>
+<p class="last">Apache Airflow is an effort undergoing incubation at The Apache
+Software Foundation (ASF), sponsored by Chris Riccomini.
+Incubation is required of all newly accepted projects until a further
+review indicates that the infrastructure, communications, and
+decision making process have stabilized in a manner consistent with
+other successful ASF projects. While incubation status is not
+necessarily a reflection of the completeness or stability of
+the code, it does indicate that the project has yet to be fully
+endorsed by the ASF.</p>
+</div>
 <p>Airflow is a platform to programmatically author, schedule and monitor
 workflows.</p>
 <p>Use airflow to author workflows as directed acyclic graphs (DAGs) of tasks.
@@ -150,7 +162,6 @@ makes it easy to visualize pipelines running in production,
 monitor progress, and troubleshoot issues when needed.</p>
 <p>When workflows are defined as code, they become more maintainable,
 versionable, testable, and collaborative.</p>
-<hr class="docutils" />
 <img alt="_images/airflow.gif" src="_images/airflow.gif" />
 <hr class="docutils" />
 <div class="section" id="principles">
@@ -324,7 +335,7 @@ unit of work and continuity.</p>
 <li class="toctree-l3"><a class="reference internal" href="code.html#baseoperator">BaseOperator</a></li>
 <li class="toctree-l3"><a class="reference internal" href="code.html#basesensoroperator">BaseSensorOperator</a></li>
 <li class="toctree-l3"><a class="reference internal" href="code.html#module-airflow.operators">Operator API</a></li>
-<li class="toctree-l3"><a class="reference internal" href="code.html#community-contributed-operators">Community-contributed Operators</a></li>
+<li class="toctree-l3"><a class="reference internal" href="code.html#module-airflow.contrib.operators">Community-contributed Operators</a></li>
 </ul>
 </li>
 <li class="toctree-l2"><a class="reference internal" href="code.html#macros">Macros</a><ul>
@@ -334,7 +345,7 @@ unit of work and continuity.</p>
 </li>
 <li class="toctree-l2"><a class="reference internal" href="code.html#models">Models</a></li>
 <li class="toctree-l2"><a class="reference internal" href="code.html#module-airflow.hooks">Hooks</a><ul>
-<li class="toctree-l3"><a class="reference internal" href="code.html#community-contributed-hooks">Community contributed hooks</a></li>
+<li class="toctree-l3"><a class="reference internal" href="code.html#module-airflow.contrib.hooks">Community contributed hooks</a></li>
 </ul>
 </li>
 <li class="toctree-l2"><a class="reference internal" href="code.html#executors">Executors</a><ul>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/502934ca/objects.inv
----------------------------------------------------------------------
diff --git a/objects.inv b/objects.inv
index ea45f19..36f1543 100644
Binary files a/objects.inv and b/objects.inv differ

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/502934ca/py-modindex.html
----------------------------------------------------------------------
diff --git a/py-modindex.html b/py-modindex.html
index 7392ccf..d377e08 100644
--- a/py-modindex.html
+++ b/py-modindex.html
@@ -156,11 +156,6 @@
      <tr class="cg-1">
        <td></td>
        <td>&nbsp;&nbsp;&nbsp;
-       <a href="code.html#module-airflow.contrib.executors"><code class="xref">airflow.contrib.executors</code></a></td><td>
-       <em></em></td></tr>
-     <tr class="cg-1">
-       <td></td>
-       <td>&nbsp;&nbsp;&nbsp;
        <a href="code.html#module-airflow.contrib.hooks"><code class="xref">airflow.contrib.hooks</code></a></td><td>
        <em></em></td></tr>
      <tr class="cg-1">

[4/6] incubator-airflow-site git commit: Adding incubator related disclaimer

[ma...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/502934ca/_modules/bigquery_hook.html
----------------------------------------------------------------------
diff --git a/_modules/bigquery_hook.html b/_modules/bigquery_hook.html
new file mode 100644
index 0000000..9478587
--- /dev/null
+++ b/_modules/bigquery_hook.html
@@ -0,0 +1,995 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>bigquery_hook &mdash; Airflow Documentation</title>
+  
+
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+    <link rel="top" title="Airflow Documentation" href="../index.html"/>
+        <link rel="up" title="Module code" href="index.html"/> 
+
+  
+  <script src="../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+                <ul>
+<li class="toctree-l1"><a class="reference internal" href="../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+        <a href="../index.html">Airflow</a>
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+  <ul class="wy-breadcrumbs">
+    <li><a href="../index.html">Docs</a> &raquo;</li>
+      
+          <li><a href="index.html">Module code</a> &raquo;</li>
+      
+    <li>bigquery_hook</li>
+      <li class="wy-breadcrumbs-aside">
+        
+          
+        
+      </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for bigquery_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+<span class="c1">#</span>
+
+<span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">This module contains a BigQuery Hook, as well as a very basic PEP 249</span>
+<span class="sd">implementation for BigQuery.</span>
+<span class="sd">&quot;&quot;&quot;</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
+<span class="kn">import</span> <span class="nn">time</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcp_api_base_hook</span> <span class="kn">import</span> <span class="n">GoogleCloudBaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="kn">import</span> <span class="n">DbApiHook</span>
+<span class="kn">from</span> <span class="nn">apiclient.discovery</span> <span class="kn">import</span> <span class="n">build</span>
+<span class="kn">from</span> <span class="nn">pandas.io.gbq</span> <span class="kn">import</span> <span class="n">GbqConnector</span><span class="p">,</span> \
+    <span class="n">_parse_data</span> <span class="k">as</span> <span class="n">gbq_parse_data</span><span class="p">,</span> \
+    <span class="n">_check_google_client_version</span> <span class="k">as</span> <span class="n">gbq_check_google_client_version</span><span class="p">,</span> \
+    <span class="n">_test_google_api_imports</span> <span class="k">as</span> <span class="n">gbq_test_google_api_imports</span>
+<span class="kn">from</span> <span class="nn">pandas.tools.merge</span> <span class="kn">import</span> <span class="n">concat</span>
+
+<span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">&quot;bigquery&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">INFO</span><span class="p">)</span>
+
+
+<div class="viewcode-block" id="BigQueryHook"><a class="viewcode-back" href="../code.html#airflow.contrib.hooks.BigQueryHook">[docs]</a><span class="k">class</span> <span class="nc">BigQueryHook</span><span class="p">(</span><span class="n">GoogleCloudBaseHook</span><span class="p">,</span> <span class="n">DbApiHook</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Interact with BigQuery. This hook uses the Google Cloud Platform</span>
+<span class="sd">    connection.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">conn_name_attr</span> <span class="o">=</span> <span class="s1">&#39;bigquery_conn_id&#39;</span>
+
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+                 <span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">&#39;bigquery_default&#39;</span><span class="p">,</span>
+                 <span class="n">delegate_to</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">BigQueryHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span>
+            <span class="n">conn_id</span><span class="o">=</span><span class="n">bigquery_conn_id</span><span class="p">,</span>
+            <span class="n">delegate_to</span><span class="o">=</span><span class="n">delegate_to</span><span class="p">)</span>
+
+<div class="viewcode-block" id="BigQueryHook.get_conn"><a class="viewcode-back" href="../code.html#airflow.contrib.hooks.BigQueryHook.get_conn">[docs]</a>    <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Returns a BigQuery PEP 249 connection object.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_service</span><span class="p">()</span>
+        <span class="n">project</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_field</span><span class="p">(</span><span class="s1">&#39;project&#39;</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">BigQueryConnection</span><span class="p">(</span><span class="n">service</span><span class="o">=</span><span class="n">service</span><span class="p">,</span> <span class="n">project_id</span><span class="o">=</span><span class="n">project</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="BigQueryHook.get_service"><a class="viewcode-back" href="../code.html#airflow.contrib.hooks.BigQueryHook.get_service">[docs]</a>    <span class="k">def</span> <span class="nf">get_service</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Returns a BigQuery service object.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">http_authorized</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_authorize</span><span class="p">()</span>
+        <span class="k">return</span> <span class="n">build</span><span class="p">(</span><span class="s1">&#39;bigquery&#39;</span><span class="p">,</span> <span class="s1">&#39;v2&#39;</span><span class="p">,</span> <span class="n">http</span><span class="o">=</span><span class="n">http_authorized</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="BigQueryHook.insert_rows"><a class="viewcode-back" href="../code.html#airflow.contrib.hooks.BigQueryHook.insert_rows">[docs]</a>    <span class="k">def</span> <span class="nf">insert_rows</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table</span><span class="p">,</span> <span class="n">rows</span><span class="p">,</span> <span class="n">target_fields</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">commit_every</span><span class="o">=</span><span class="mi">1000</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Insertion is currently unsupported. Theoretically, you could use</span>
+<span class="sd">        BigQuery&#39;s streaming API to insert rows into a table, but this hasn&#39;t</span>
+<span class="sd">        been implemented.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">()</span></div>
+
+<div class="viewcode-block" id="BigQueryHook.get_pandas_df"><a class="viewcode-back" href="../code.html#airflow.contrib.hooks.BigQueryHook.get_pandas_df">[docs]</a>    <span class="k">def</span> <span class="nf">get_pandas_df</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bql</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Returns a Pandas DataFrame for the results produced by a BigQuery</span>
+<span class="sd">        query. The DbApiHook method must be overridden because Pandas</span>
+<span class="sd">        doesn&#39;t support PEP 249 connections, except for SQLite. See:</span>
+
+<span class="sd">        https://github.com/pydata/pandas/blob/master/pandas/io/sql.py#L447</span>
+<span class="sd">        https://github.com/pydata/pandas/issues/6900</span>
+
+<span class="sd">        :param bql: The BigQuery SQL to execute.</span>
+<span class="sd">        :type bql: string</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_service</span><span class="p">()</span>
+        <span class="n">project</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_field</span><span class="p">(</span><span class="s1">&#39;project&#39;</span><span class="p">)</span>
+        <span class="n">connector</span> <span class="o">=</span> <span class="n">BigQueryPandasConnector</span><span class="p">(</span><span class="n">project</span><span class="p">,</span> <span class="n">service</span><span class="p">)</span>
+        <span class="n">schema</span><span class="p">,</span> <span class="n">pages</span> <span class="o">=</span> <span class="n">connector</span><span class="o">.</span><span class="n">run_query</span><span class="p">(</span><span class="n">bql</span><span class="p">)</span>
+        <span class="n">dataframe_list</span> <span class="o">=</span> <span class="p">[]</span>
+
+        <span class="k">while</span> <span class="nb">len</span><span class="p">(</span><span class="n">pages</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
+            <span class="n">page</span> <span class="o">=</span> <span class="n">pages</span><span class="o">.</span><span class="n">pop</span><span class="p">()</span>
+            <span class="n">dataframe_list</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">gbq_parse_data</span><span class="p">(</span><span class="n">schema</span><span class="p">,</span> <span class="n">page</span><span class="p">))</span>
+
+        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">dataframe_list</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">concat</span><span class="p">(</span><span class="n">dataframe_list</span><span class="p">,</span> <span class="n">ignore_index</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">gbq_parse_data</span><span class="p">(</span><span class="n">schema</span><span class="p">,</span> <span class="p">[])</span></div></div>
+
+
+<span class="k">class</span> <span class="nc">BigQueryPandasConnector</span><span class="p">(</span><span class="n">GbqConnector</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    This connector behaves identically to GbqConnector (from Pandas), except</span>
+<span class="sd">    that it allows the service to be injected, and disables a call to</span>
+<span class="sd">    self.get_credentials(). This allows Airflow to use BigQuery with Pandas</span>
+<span class="sd">    without forcing a three legged OAuth connection. Instead, we can inject</span>
+<span class="sd">    service account credentials into the binding.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">service</span><span class="p">,</span> <span class="n">reauth</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="bp">False</span><span class="p">):</span>
+        <span class="n">gbq_check_google_client_version</span><span class="p">()</span>
+        <span class="n">gbq_test_google_api_imports</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">reauth</span> <span class="o">=</span> <span class="n">reauth</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">service</span> <span class="o">=</span> <span class="n">service</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">=</span> <span class="n">verbose</span>
+
+
+<span class="k">class</span> <span class="nc">BigQueryConnection</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    BigQuery does not have a notion of a persistent connection. Thus, these</span>
+<span class="sd">    objects are small stateless factories for cursors, which do all the real</span>
+<span class="sd">    work.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_args</span> <span class="o">=</span> <span class="n">args</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_kwargs</span> <span class="o">=</span> <span class="n">kwargs</span>
+
+    <span class="k">def</span> <span class="nf">close</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot; BigQueryConnection does not have anything to close. &quot;&quot;&quot;</span>
+        <span class="k">pass</span>
+
+    <span class="k">def</span> <span class="nf">commit</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot; BigQueryConnection does not support transactions. &quot;&quot;&quot;</span>
+        <span class="k">pass</span>
+
+    <span class="k">def</span> <span class="nf">cursor</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot; Return a new :py:class:`Cursor` object using the connection. &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="n">BigQueryCursor</span><span class="p">(</span><span class="o">*</span><span class="bp">self</span><span class="o">.</span><span class="n">_args</span><span class="p">,</span> <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">_kwargs</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">rollback</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span>
+            <span class="s2">&quot;BigQueryConnection does not have transactions&quot;</span><span class="p">)</span>
+
+
+<span class="k">class</span> <span class="nc">BigQueryBaseCursor</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    The BigQuery base cursor contains helper methods to execute queries against</span>
+<span class="sd">    BigQuery. The methods can be used directly by operators, in cases where a</span>
+<span class="sd">    PEP 249 cursor isn&#39;t needed.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">service</span><span class="p">,</span> <span class="n">project_id</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">service</span> <span class="o">=</span> <span class="n">service</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span>
+
+    <span class="k">def</span> <span class="nf">run_query</span><span class="p">(</span>
+            <span class="bp">self</span><span class="p">,</span> <span class="n">bql</span><span class="p">,</span> <span class="n">destination_dataset_table</span> <span class="o">=</span> <span class="bp">False</span><span class="p">,</span>
+            <span class="n">write_disposition</span> <span class="o">=</span> <span class="s1">&#39;WRITE_EMPTY&#39;</span><span class="p">,</span>
+            <span class="n">allow_large_results</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
+            <span class="n">udf_config</span> <span class="o">=</span> <span class="bp">False</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Executes a BigQuery SQL query. Optionally persists results in a BigQuery</span>
+<span class="sd">        table. See here:</span>
+
+<span class="sd">        https://cloud.google.com/bigquery/docs/reference/v2/jobs</span>
+
+<span class="sd">        For more details about these parameters.</span>
+
+<span class="sd">        :param bql: The BigQuery SQL to execute.</span>
+<span class="sd">        :type bql: string</span>
+<span class="sd">        :param destination_dataset_table: The dotted &lt;dataset&gt;.&lt;table&gt;</span>
+<span class="sd">            BigQuery table to save the query results.</span>
+<span class="sd">        :param write_disposition: What to do if the table already exists in</span>
+<span class="sd">            BigQuery.</span>
+<span class="sd">        :param allow_large_results: Whether to allow large results.</span>
+<span class="sd">        :type allow_large_results: boolean</span>
+<span class="sd">        :param udf_config: The User Defined Function configuration for the query.</span>
+<span class="sd">            See https://cloud.google.com/bigquery/user-defined-functions for details.</span>
+<span class="sd">        :type udf_config: list</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">configuration</span> <span class="o">=</span> <span class="p">{</span>
+            <span class="s1">&#39;query&#39;</span><span class="p">:</span> <span class="p">{</span>
+                <span class="s1">&#39;query&#39;</span><span class="p">:</span> <span class="n">bql</span><span class="p">,</span>
+            <span class="p">}</span>
+        <span class="p">}</span>
+
+        <span class="k">if</span> <span class="n">destination_dataset_table</span><span class="p">:</span>
+            <span class="k">assert</span> <span class="s1">&#39;.&#39;</span> <span class="ow">in</span> <span class="n">destination_dataset_table</span><span class="p">,</span> <span class="p">(</span>
+                <span class="s1">&#39;Expected destination_dataset_table in the format of &#39;</span>
+                <span class="s1">&#39;&lt;dataset&gt;.&lt;table&gt;. Got: {}&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">destination_dataset_table</span><span class="p">)</span>
+            <span class="n">destination_dataset</span><span class="p">,</span> <span class="n">destination_table</span> <span class="o">=</span> \
+                <span class="n">destination_dataset_table</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
+            <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;query&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">({</span>
+                <span class="s1">&#39;allowLargeResults&#39;</span><span class="p">:</span> <span class="n">allow_large_results</span><span class="p">,</span>
+                <span class="s1">&#39;writeDisposition&#39;</span><span class="p">:</span> <span class="n">write_disposition</span><span class="p">,</span>
+                <span class="s1">&#39;destinationTable&#39;</span><span class="p">:</span> <span class="p">{</span>
+                    <span class="s1">&#39;projectId&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+                    <span class="s1">&#39;datasetId&#39;</span><span class="p">:</span> <span class="n">destination_dataset</span><span class="p">,</span>
+                    <span class="s1">&#39;tableId&#39;</span><span class="p">:</span> <span class="n">destination_table</span><span class="p">,</span>
+                <span class="p">}</span>
+            <span class="p">})</span>
+        <span class="k">if</span> <span class="n">udf_config</span><span class="p">:</span>
+            <span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">udf_config</span><span class="p">,</span> <span class="nb">list</span><span class="p">)</span>
+            <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;query&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">({</span>
+                <span class="s1">&#39;userDefinedFunctionResources&#39;</span><span class="p">:</span> <span class="n">udf_config</span>
+            <span class="p">})</span>
+
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_with_configuration</span><span class="p">(</span><span class="n">configuration</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">run_extract</span><span class="p">(</span>  <span class="c1"># noqa</span>
+            <span class="bp">self</span><span class="p">,</span> <span class="n">source_project_dataset_table</span><span class="p">,</span> <span class="n">destination_cloud_storage_uris</span><span class="p">,</span>
+            <span class="n">compression</span><span class="o">=</span><span class="s1">&#39;NONE&#39;</span><span class="p">,</span> <span class="n">export_format</span><span class="o">=</span><span class="s1">&#39;CSV&#39;</span><span class="p">,</span> <span class="n">field_delimiter</span><span class="o">=</span><span class="s1">&#39;,&#39;</span><span class="p">,</span>
+            <span class="n">print_header</span><span class="o">=</span><span class="bp">True</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Executes a BigQuery extract command to copy data from BigQuery to</span>
+<span class="sd">        Google Cloud Storage. See here:</span>
+
+<span class="sd">        https://cloud.google.com/bigquery/docs/reference/v2/jobs</span>
+
+<span class="sd">        For more details about these parameters.</span>
+
+<span class="sd">        :param source_project_dataset_table: The dotted &lt;dataset&gt;.&lt;table&gt;</span>
+<span class="sd">            BigQuery table to use as the source data.</span>
+<span class="sd">        :type source_project_dataset_table: string</span>
+<span class="sd">        :param destination_cloud_storage_uris: The destination Google Cloud</span>
+<span class="sd">            Storage URI (e.g. gs://some-bucket/some-file.txt). Follows</span>
+<span class="sd">            convention defined here:</span>
+<span class="sd">            https://cloud.google.com/bigquery/exporting-data-from-bigquery#exportingmultiple</span>
+<span class="sd">        :type destination_cloud_storage_uris: list</span>
+<span class="sd">        :param compression: Type of compression to use.</span>
+<span class="sd">        :type compression: string</span>
+<span class="sd">        :param export_format: File format to export.</span>
+<span class="sd">        :type export_format: string</span>
+<span class="sd">        :param field_delimiter: The delimiter to use when extracting to a CSV.</span>
+<span class="sd">        :type field_delimiter: string</span>
+<span class="sd">        :param print_header: Whether to print a header for a CSV file extract.</span>
+<span class="sd">        :type print_header: boolean</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">source_project</span><span class="p">,</span> <span class="n">source_dataset</span><span class="p">,</span> <span class="n">source_table</span> <span class="o">=</span> \
+            <span class="bp">self</span><span class="o">.</span><span class="n">_split_project_dataset_table_input</span><span class="p">(</span>
+                <span class="s1">&#39;source_project_dataset_table&#39;</span><span class="p">,</span> <span class="n">source_project_dataset_table</span><span class="p">)</span>
+        <span class="n">configuration</span> <span class="o">=</span> <span class="p">{</span>
+            <span class="s1">&#39;extract&#39;</span><span class="p">:</span> <span class="p">{</span>
+                <span class="s1">&#39;sourceTable&#39;</span><span class="p">:</span> <span class="p">{</span>
+                    <span class="s1">&#39;projectId&#39;</span><span class="p">:</span> <span class="n">source_project</span><span class="p">,</span>
+                    <span class="s1">&#39;datasetId&#39;</span><span class="p">:</span> <span class="n">source_dataset</span><span class="p">,</span>
+                    <span class="s1">&#39;tableId&#39;</span><span class="p">:</span> <span class="n">source_table</span><span class="p">,</span>
+                <span class="p">},</span>
+                <span class="s1">&#39;compression&#39;</span><span class="p">:</span> <span class="n">compression</span><span class="p">,</span>
+                <span class="s1">&#39;destinationUris&#39;</span><span class="p">:</span> <span class="n">destination_cloud_storage_uris</span><span class="p">,</span>
+                <span class="s1">&#39;destinationFormat&#39;</span><span class="p">:</span> <span class="n">export_format</span><span class="p">,</span>
+            <span class="p">}</span>
+        <span class="p">}</span>
+
+        <span class="k">if</span> <span class="n">export_format</span> <span class="o">==</span> <span class="s1">&#39;CSV&#39;</span><span class="p">:</span>
+            <span class="c1"># Only set fieldDelimiter and printHeader fields if using CSV.</span>
+            <span class="c1"># Google does not like it if you set these fields for other export</span>
+            <span class="c1"># formats.</span>
+            <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;extract&#39;</span><span class="p">][</span><span class="s1">&#39;fieldDelimiter&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">field_delimiter</span>
+            <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;extract&#39;</span><span class="p">][</span><span class="s1">&#39;printHeader&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">print_header</span>
+
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_with_configuration</span><span class="p">(</span><span class="n">configuration</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">run_copy</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+                 <span class="n">source_project_dataset_tables</span><span class="p">,</span>
+                 <span class="n">destination_project_dataset_table</span><span class="p">,</span>
+                 <span class="n">write_disposition</span><span class="o">=</span><span class="s1">&#39;WRITE_EMPTY&#39;</span><span class="p">,</span>
+                 <span class="n">create_disposition</span><span class="o">=</span><span class="s1">&#39;CREATE_IF_NEEDED&#39;</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Executes a BigQuery copy command to copy data from one BigQuery table</span>
+<span class="sd">        to another. See here:</span>
+
+<span class="sd">        https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.copy</span>
+
+<span class="sd">        For more details about these parameters.</span>
+
+<span class="sd">        :param source_project_dataset_tables: One or more dotted</span>
+<span class="sd">            (&lt;project&gt;.)&lt;dataset&gt;.&lt;table&gt;</span>
+<span class="sd">            BigQuery tables to use as the source data. Use a list if there are</span>
+<span class="sd">            multiple source tables.</span>
+<span class="sd">            If &lt;project&gt; is not included, project will be the project defined</span>
+<span class="sd">            in the connection json.</span>
+<span class="sd">        :type source_project_dataset_tables: list|string</span>
+<span class="sd">        :param destination_project_dataset_table: The destination BigQuery</span>
+<span class="sd">            table. Format is: &lt;project&gt;.&lt;dataset&gt;.&lt;table&gt;</span>
+<span class="sd">        :type destination_project_dataset_table: string</span>
+<span class="sd">        :param write_disposition: The write disposition if the table already exists.</span>
+<span class="sd">        :type write_disposition: string</span>
+<span class="sd">        :param create_disposition: The create disposition if the table doesn&#39;t exist.</span>
+<span class="sd">        :type create_disposition: string</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">source_project_dataset_tables</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="p">[</span><span class="n">source_project_dataset_tables</span><span class="p">]</span>
+            <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">source_project_dataset_tables</span><span class="p">,</span> <span class="nb">list</span><span class="p">)</span>
+            <span class="k">else</span> <span class="n">source_project_dataset_tables</span><span class="p">)</span>
+
+        <span class="n">source_project_dataset_tables_fixup</span> <span class="o">=</span> <span class="p">[]</span>
+        <span class="k">for</span> <span class="n">source_project_dataset_table</span> <span class="ow">in</span> <span class="n">source_project_dataset_tables</span><span class="p">:</span>
+            <span class="n">source_project</span><span class="p">,</span> <span class="n">source_dataset</span><span class="p">,</span> <span class="n">source_table</span> <span class="o">=</span> \
+                <span class="bp">self</span><span class="o">.</span><span class="n">_split_project_dataset_table_input</span><span class="p">(</span>
+                    <span class="s1">&#39;source_project_dataset_table&#39;</span><span class="p">,</span> <span class="n">source_project_dataset_table</span><span class="p">)</span>
+            <span class="n">source_project_dataset_tables_fixup</span><span class="o">.</span><span class="n">append</span><span class="p">({</span>
+                    <span class="s1">&#39;projectId&#39;</span><span class="p">:</span> <span class="n">source_project</span><span class="p">,</span>
+                    <span class="s1">&#39;datasetId&#39;</span><span class="p">:</span> <span class="n">source_dataset</span><span class="p">,</span>
+                    <span class="s1">&#39;tableId&#39;</span><span class="p">:</span> <span class="n">source_table</span>
+                <span class="p">})</span>
+
+        <span class="k">assert</span> <span class="mi">3</span> <span class="o">==</span> <span class="nb">len</span><span class="p">(</span><span class="n">destination_project_dataset_table</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">)),</span> <span class="p">(</span>
+            <span class="s1">&#39;Expected destination_project_dataset_table in the format of &#39;</span>
+            <span class="s1">&#39;&lt;project&gt;.&lt;dataset&gt;.&lt;table&gt;. &#39;</span>
+            <span class="s1">&#39;Got: {}&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">destination_project_dataset_table</span><span class="p">)</span>
+
+        <span class="n">destination_project</span><span class="p">,</span> <span class="n">destination_dataset</span><span class="p">,</span> <span class="n">destination_table</span> <span class="o">=</span> \
+            <span class="n">destination_project_dataset_table</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
+        <span class="n">configuration</span> <span class="o">=</span> <span class="p">{</span>
+            <span class="s1">&#39;copy&#39;</span><span class="p">:</span> <span class="p">{</span>
+                <span class="s1">&#39;createDisposition&#39;</span><span class="p">:</span> <span class="n">create_disposition</span><span class="p">,</span>
+                <span class="s1">&#39;writeDisposition&#39;</span><span class="p">:</span> <span class="n">write_disposition</span><span class="p">,</span>
+                <span class="s1">&#39;sourceTables&#39;</span><span class="p">:</span> <span class="n">source_project_dataset_tables_fixup</span><span class="p">,</span>
+                <span class="s1">&#39;destinationTable&#39;</span><span class="p">:</span> <span class="p">{</span>
+                    <span class="s1">&#39;projectId&#39;</span><span class="p">:</span> <span class="n">destination_project</span><span class="p">,</span>
+                    <span class="s1">&#39;datasetId&#39;</span><span class="p">:</span> <span class="n">destination_dataset</span><span class="p">,</span>
+                    <span class="s1">&#39;tableId&#39;</span><span class="p">:</span> <span class="n">destination_table</span>
+                <span class="p">}</span>
+            <span class="p">}</span>
+        <span class="p">}</span>
+
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_with_configuration</span><span class="p">(</span><span class="n">configuration</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">run_load</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+                 <span class="n">destination_project_dataset_table</span><span class="p">,</span>
+                 <span class="n">schema_fields</span><span class="p">,</span> <span class="n">source_uris</span><span class="p">,</span>
+                 <span class="n">source_format</span><span class="o">=</span><span class="s1">&#39;CSV&#39;</span><span class="p">,</span>
+                 <span class="n">create_disposition</span><span class="o">=</span><span class="s1">&#39;CREATE_IF_NEEDED&#39;</span><span class="p">,</span>
+                 <span class="n">skip_leading_rows</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
+                 <span class="n">write_disposition</span><span class="o">=</span><span class="s1">&#39;WRITE_EMPTY&#39;</span><span class="p">,</span>
+                 <span class="n">field_delimiter</span><span class="o">=</span><span class="s1">&#39;,&#39;</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Executes a BigQuery load command to load data from Google Cloud Storage</span>
+<span class="sd">        to BigQuery. See here:</span>
+
+<span class="sd">        https://cloud.google.com/bigquery/docs/reference/v2/jobs</span>
+
+<span class="sd">        For more details about these parameters.</span>
+
+<span class="sd">        :param destination_project_dataset_table:</span>
+<span class="sd">            The dotted (&lt;project&gt;.)&lt;dataset&gt;.&lt;table&gt; BigQuery table to load data into.</span>
+<span class="sd">            If &lt;project&gt; is not included, project will be the project defined in</span>
+<span class="sd">            the connection json.</span>
+<span class="sd">        :type destination_project_dataset_table: string</span>
+<span class="sd">        :param schema_fields: The schema field list as defined here:</span>
+<span class="sd">            https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.load</span>
+<span class="sd">        :type schema_fields: list</span>
+<span class="sd">        :param source_uris: The source Google Cloud</span>
+<span class="sd">            Storage URI (e.g. gs://some-bucket/some-file.txt). A single wild</span>
+<span class="sd">            per-object name can be used.</span>
+<span class="sd">        :type source_uris: list</span>
+<span class="sd">        :param source_format: File format to export.</span>
+<span class="sd">        :type source_format: string</span>
+<span class="sd">        :param create_disposition: The create disposition if the table doesn&#39;t exist.</span>
+<span class="sd">        :type create_disposition: string</span>
+<span class="sd">        :param skip_leading_rows: Number of rows to skip when loading from a CSV.</span>
+<span class="sd">        :type skip_leading_rows: int</span>
+<span class="sd">        :param write_disposition: The write disposition if the table already exists.</span>
+<span class="sd">        :type write_disposition: string</span>
+<span class="sd">        :param field_delimiter: The delimiter to use when loading from a CSV.</span>
+<span class="sd">        :type field_delimiter: string</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">destination_project</span><span class="p">,</span> <span class="n">destination_dataset</span><span class="p">,</span> <span class="n">destination_table</span> <span class="o">=</span> \
+            <span class="bp">self</span><span class="o">.</span><span class="n">_split_project_dataset_table_input</span><span class="p">(</span>
+                <span class="s1">&#39;destination_project_dataset_table&#39;</span><span class="p">,</span> <span class="n">destination_project_dataset_table</span><span class="p">)</span>
+
+        <span class="n">configuration</span> <span class="o">=</span> <span class="p">{</span>
+            <span class="s1">&#39;load&#39;</span><span class="p">:</span> <span class="p">{</span>
+                <span class="s1">&#39;createDisposition&#39;</span><span class="p">:</span> <span class="n">create_disposition</span><span class="p">,</span>
+                <span class="s1">&#39;destinationTable&#39;</span><span class="p">:</span> <span class="p">{</span>
+                    <span class="s1">&#39;projectId&#39;</span><span class="p">:</span> <span class="n">destination_project</span><span class="p">,</span>
+                    <span class="s1">&#39;datasetId&#39;</span><span class="p">:</span> <span class="n">destination_dataset</span><span class="p">,</span>
+                    <span class="s1">&#39;tableId&#39;</span><span class="p">:</span> <span class="n">destination_table</span><span class="p">,</span>
+                <span class="p">},</span>
+                <span class="s1">&#39;schema&#39;</span><span class="p">:</span> <span class="p">{</span>
+                    <span class="s1">&#39;fields&#39;</span><span class="p">:</span> <span class="n">schema_fields</span>
+                <span class="p">},</span>
+                <span class="s1">&#39;sourceFormat&#39;</span><span class="p">:</span> <span class="n">source_format</span><span class="p">,</span>
+                <span class="s1">&#39;sourceUris&#39;</span><span class="p">:</span> <span class="n">source_uris</span><span class="p">,</span>
+                <span class="s1">&#39;writeDisposition&#39;</span><span class="p">:</span> <span class="n">write_disposition</span><span class="p">,</span>
+            <span class="p">}</span>
+        <span class="p">}</span>
+
+        <span class="k">if</span> <span class="n">source_format</span> <span class="o">==</span> <span class="s1">&#39;CSV&#39;</span><span class="p">:</span>
+            <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;load&#39;</span><span class="p">][</span><span class="s1">&#39;skipLeadingRows&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">skip_leading_rows</span>
+            <span class="n">configuration</span><span class="p">[</span><span class="s1">&#39;load&#39;</span><span class="p">][</span><span class="s1">&#39;fieldDelimiter&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">field_delimiter</span>
+
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_with_configuration</span><span class="p">(</span><span class="n">configuration</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_split_project_dataset_table_input</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">var_name</span><span class="p">,</span> <span class="n">project_dataset_table</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        :param var_name: the name of the variable input, for logging and erroring purposes.</span>
+<span class="sd">        :type var_name: str</span>
+<span class="sd">        :param project_dataset_table: input string in (&lt;project&gt;.)&lt;dataset&gt;.&lt;project&gt; format.</span>
+<span class="sd">            if project is not included in the string, self.project_id will be returned in the tuple.</span>
+<span class="sd">        :type project_dataset_table: str</span>
+<span class="sd">        :return: (project, dataset, table) tuple</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">table_split</span> <span class="o">=</span> <span class="n">project_dataset_table</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">)</span>
+        <span class="k">assert</span> <span class="nb">len</span><span class="p">(</span><span class="n">table_split</span><span class="p">)</span> <span class="o">==</span> <span class="mi">2</span> <span class="ow">or</span> <span class="nb">len</span><span class="p">(</span><span class="n">table_split</span><span class="p">)</span> <span class="o">==</span> <span class="mi">3</span><span class="p">,</span> <span class="p">(</span>
+            <span class="s1">&#39;Expected {var} in the format of (&lt;project.)&lt;dataset&gt;.&lt;table&gt;, &#39;</span>
+            <span class="s1">&#39;got {input}&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">var</span><span class="o">=</span><span class="n">var_name</span><span class="p">,</span> <span class="nb">input</span><span class="o">=</span><span class="n">project_dataset_table</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">table_split</span><span class="p">)</span> <span class="o">==</span> <span class="mi">2</span><span class="p">:</span>
+            <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;project not included in {var}: {input}; using project &quot;{project}&quot;&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">var</span><span class="o">=</span><span class="n">var_name</span><span class="p">,</span> <span class="nb">input</span><span class="o">=</span><span class="n">project_dataset_table</span><span class="p">,</span> <span class="n">project</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">))</span>
+            <span class="n">dataset</span><span class="p">,</span> <span class="n">table</span> <span class="o">=</span> <span class="n">table_split</span>
+            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">dataset</span><span class="p">,</span> <span class="n">table</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">project</span><span class="p">,</span> <span class="n">dataset</span><span class="p">,</span> <span class="n">table</span> <span class="o">=</span> <span class="n">table_split</span>
+            <span class="k">return</span> <span class="n">project</span><span class="p">,</span> <span class="n">dataset</span><span class="p">,</span> <span class="n">table</span>
+
+    <span class="k">def</span> <span class="nf">run_with_configuration</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">configuration</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Executes a BigQuery SQL query. See here:</span>
+
+<span class="sd">        https://cloud.google.com/bigquery/docs/reference/v2/jobs</span>
+
+<span class="sd">        For more details about the configuration parameter.</span>
+
+<span class="sd">        :param configuration: The configuration parameter maps directly to</span>
+<span class="sd">            BigQuery&#39;s configuration field in the job object. See</span>
+<span class="sd">            https://cloud.google.com/bigquery/docs/reference/v2/jobs for</span>
+<span class="sd">            details.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">jobs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">jobs</span><span class="p">()</span>
+        <span class="n">job_data</span> <span class="o">=</span> <span class="p">{</span>
+            <span class="s1">&#39;configuration&#39;</span><span class="p">:</span> <span class="n">configuration</span>
+        <span class="p">}</span>
+
+        <span class="c1"># Send query and wait for reply.</span>
+        <span class="n">query_reply</span> <span class="o">=</span> <span class="n">jobs</span> \
+            <span class="o">.</span><span class="n">insert</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="n">job_data</span><span class="p">)</span> \
+            <span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+        <span class="n">job_id</span> <span class="o">=</span> <span class="n">query_reply</span><span class="p">[</span><span class="s1">&#39;jobReference&#39;</span><span class="p">][</span><span class="s1">&#39;jobId&#39;</span><span class="p">]</span>
+        <span class="n">job</span> <span class="o">=</span> <span class="n">jobs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">jobId</span><span class="o">=</span><span class="n">job_id</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+
+        <span class="c1"># Wait for query to finish.</span>
+        <span class="k">while</span> <span class="ow">not</span> <span class="n">job</span><span class="p">[</span><span class="s1">&#39;status&#39;</span><span class="p">][</span><span class="s1">&#39;state&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="s1">&#39;DONE&#39;</span><span class="p">:</span>
+            <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Waiting for job to complete: </span><span class="si">%s</span><span class="s1">, </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job_id</span><span class="p">)</span>
+            <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">5</span><span class="p">)</span>
+            <span class="n">job</span> <span class="o">=</span> <span class="n">jobs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">jobId</span><span class="o">=</span><span class="n">job_id</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+
+        <span class="c1"># Check if job had errors.</span>
+        <span class="k">if</span> <span class="s1">&#39;errorResult&#39;</span> <span class="ow">in</span> <span class="n">job</span><span class="p">[</span><span class="s1">&#39;status&#39;</span><span class="p">]:</span>
+            <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span>
+                <span class="s1">&#39;BigQuery job failed. Final error was: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="n">job</span><span class="p">[</span><span class="s1">&#39;status&#39;</span><span class="p">][</span><span class="s1">&#39;errorResult&#39;</span><span class="p">])</span>
+
+        <span class="k">return</span> <span class="n">job_id</span>
+
+    <span class="k">def</span> <span class="nf">get_schema</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Get the schema for a given datset.table.</span>
+<span class="sd">        see https://cloud.google.com/bigquery/docs/reference/v2/tables#resource</span>
+
+<span class="sd">        :param dataset_id: the dataset ID of the requested table</span>
+<span class="sd">        :param table_id: the table ID of the requested table</span>
+<span class="sd">        :return: a table schema</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">tables_resource</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span> \
+            <span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span> <span class="n">tableId</span><span class="o">=</span><span class="n">table_id</span><span class="p">)</span> \
+            <span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+        <span class="k">return</span> <span class="n">tables_resource</span><span class="p">[</span><span class="s1">&#39;schema&#39;</span><span class="p">]</span>
+
+    <span class="k">def</span> <span class="nf">get_tabledata</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">,</span>
+                      <span class="n">max_results</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">page_token</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">start_index</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Get the data of a given dataset.table.</span>
+<span class="sd">        see https://cloud.google.com/bigquery/docs/reference/v2/tabledata/list</span>
+
+<span class="sd">        :param dataset_id: the dataset ID of the requested table.</span>
+<span class="sd">        :param table_id: the table ID of the requested table.</span>
+<span class="sd">        :param max_results: the maximum results to return.</span>
+<span class="sd">        :param page_token: page token, returned from a previous call,</span>
+<span class="sd">            identifying the result set.</span>
+<span class="sd">        :param start_index: zero based index of the starting row to read.</span>
+<span class="sd">        :return: map containing the requested rows.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">optional_params</span> <span class="o">=</span> <span class="p">{}</span>
+        <span class="k">if</span> <span class="n">max_results</span><span class="p">:</span>
+            <span class="n">optional_params</span><span class="p">[</span><span class="s1">&#39;maxResults&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">max_results</span>
+        <span class="k">if</span> <span class="n">page_token</span><span class="p">:</span>
+            <span class="n">optional_params</span><span class="p">[</span><span class="s1">&#39;pageToken&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">page_token</span>
+        <span class="k">if</span> <span class="n">start_index</span><span class="p">:</span>
+            <span class="n">optional_params</span><span class="p">[</span><span class="s1">&#39;startIndex&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">start_index</span>
+        <span class="k">return</span> <span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tabledata</span><span class="p">()</span>
+            <span class="o">.</span><span class="n">list</span><span class="p">(</span>
+                <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
+                <span class="n">tableId</span><span class="o">=</span><span class="n">table_id</span><span class="p">,</span> <span class="o">**</span><span class="n">optional_params</span><span class="p">)</span>
+            <span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+        <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">run_table_upsert</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_resource</span><span class="p">,</span> <span class="n">project_id</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        creates a new, empty table in the dataset;</span>
+<span class="sd">        If the table already exists, update the existing table.</span>
+<span class="sd">        Since BigQuery does not natively allow table upserts, this is not an</span>
+<span class="sd">        atomic operation.</span>
+<span class="sd">        :param dataset_id: the dataset to upsert the table into.</span>
+<span class="sd">        :type dataset_id: str</span>
+<span class="sd">        :param table_resource: a table resource. see https://cloud.google.com/bigquery/docs/reference/v2/tables#resource</span>
+<span class="sd">        :type table_resource: dict</span>
+<span class="sd">        :param project_id: the project to upsert the table into.  If None,</span>
+<span class="sd">        project will be self.project_id.</span>
+<span class="sd">        :return:</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="c1"># check to see if the table exists</span>
+        <span class="n">table_id</span> <span class="o">=</span> <span class="n">table_resource</span><span class="p">[</span><span class="s1">&#39;tableReference&#39;</span><span class="p">][</span><span class="s1">&#39;tableId&#39;</span><span class="p">]</span>
+        <span class="n">table_exists</span> <span class="o">=</span> <span class="bp">False</span>
+        <span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span> <span class="k">if</span> <span class="n">project_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span>
+        <span class="n">tables_list_resp</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">list</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
+                                                      <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+        <span class="k">if</span> <span class="s1">&#39;tables&#39;</span> <span class="ow">in</span> <span class="n">tables_list_resp</span><span class="p">:</span>
+            <span class="k">for</span> <span class="n">table</span> <span class="ow">in</span> <span class="n">tables_list_resp</span><span class="p">[</span><span class="s1">&#39;tables&#39;</span><span class="p">]:</span>
+                <span class="k">if</span> <span class="n">table</span><span class="p">[</span><span class="s1">&#39;tableReference&#39;</span><span class="p">][</span><span class="s1">&#39;tableId&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="n">table_id</span><span class="p">:</span>
+                    <span class="n">table_exists</span> <span class="o">=</span> <span class="bp">True</span>
+                    <span class="k">break</span>
+
+        <span class="c1"># do update if table exists</span>
+        <span class="k">if</span> <span class="n">table_exists</span><span class="p">:</span>
+            <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1"> exists, updating.&#39;</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">)</span>
+            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
+                                                <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
+                                                <span class="n">tableId</span><span class="o">=</span><span class="n">table_id</span><span class="p">,</span>
+                                                <span class="n">body</span><span class="o">=</span><span class="n">table_resource</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+        <span class="c1"># do insert if table does not exist</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1"> does not exist. creating.&#39;</span><span class="p">,</span> <span class="n">project_id</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">table_id</span><span class="p">)</span>
+            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">tables</span><span class="p">()</span><span class="o">.</span><span class="n">insert</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="n">project_id</span><span class="p">,</span>
+                                                <span class="n">datasetId</span><span class="o">=</span><span class="n">dataset_id</span><span class="p">,</span>
+                                                <span class="n">body</span><span class="o">=</span><span class="n">table_resource</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">run_grant_dataset_view_access</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+                                      <span class="n">source_dataset</span><span class="p">,</span>
+                                      <span class="n">view_dataset</span><span class="p">,</span>
+                                      <span class="n">view_table</span><span class="p">,</span>
+                                      <span class="n">source_project</span> <span class="o">=</span> <span class="bp">None</span><span class="p">,</span>
+                                      <span class="n">view_project</span> <span class="o">=</span> <span class="bp">None</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Grant authorized view access of a dataset to a view table.</span>
+<span class="sd">        If this view has already been granted access to the dataset, do nothing.</span>
+<span class="sd">        This method is not atomic.  Running it may clobber a simultaneous update.</span>
+<span class="sd">        :param source_dataset: the source dataset</span>
+<span class="sd">        :type source_dataset: str</span>
+<span class="sd">        :param view_dataset: the dataset that the view is in</span>
+<span class="sd">        :type view_dataset: str</span>
+<span class="sd">        :param view_table: the table of the view</span>
+<span class="sd">        :type view_table: str</span>
+<span class="sd">        :param source_project: the project of the source dataset. If None,</span>
+<span class="sd">        self.project_id will be used.</span>
+<span class="sd">        :type source_project: str</span>
+<span class="sd">        :param view_project: the project that the view is in. If None,</span>
+<span class="sd">        self.project_id will be used.</span>
+<span class="sd">        :type view_project: str</span>
+<span class="sd">        :return: the datasets resource of the source dataset.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+
+        <span class="c1"># Apply default values to projects</span>
+        <span class="n">source_project</span> <span class="o">=</span> <span class="n">source_project</span> <span class="k">if</span> <span class="n">source_project</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span>
+        <span class="n">view_project</span> <span class="o">=</span> <span class="n">view_project</span> <span class="k">if</span> <span class="n">view_project</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span>
+
+        <span class="c1"># we don&#39;t want to clobber any existing accesses, so we have to get</span>
+        <span class="c1"># info on the dataset before we can add view access</span>
+        <span class="n">source_dataset_resource</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">datasets</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="n">source_project</span><span class="p">,</span>
+                                                              <span class="n">datasetId</span><span class="o">=</span><span class="n">source_dataset</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+        <span class="n">access</span> <span class="o">=</span> <span class="n">source_dataset_resource</span><span class="p">[</span><span class="s1">&#39;access&#39;</span><span class="p">]</span> <span class="k">if</span> <span class="s1">&#39;access&#39;</span> <span class="ow">in</span> <span class="n">source_dataset_resource</span> <span class="k">else</span> <span class="p">[]</span>
+        <span class="n">view_access</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;view&#39;</span><span class="p">:</span> <span class="p">{</span><span class="s1">&#39;projectId&#39;</span><span class="p">:</span> <span class="n">view_project</span><span class="p">,</span>
+                                <span class="s1">&#39;datasetId&#39;</span><span class="p">:</span> <span class="n">view_dataset</span><span class="p">,</span>
+                                <span class="s1">&#39;tableId&#39;</span><span class="p">:</span> <span class="n">view_table</span><span class="p">}}</span>
+        <span class="c1"># check to see if the view we want to add already exists.</span>
+        <span class="k">if</span> <span class="n">view_access</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">access</span><span class="p">:</span>
+            <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;granting table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1"> authorized view access to </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1"> dataset.&#39;</span><span class="p">,</span>
+                         <span class="n">view_project</span><span class="p">,</span> <span class="n">view_dataset</span><span class="p">,</span> <span class="n">view_table</span><span class="p">,</span>
+                         <span class="n">source_project</span><span class="p">,</span> <span class="n">source_dataset</span><span class="p">)</span>
+            <span class="n">access</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">view_access</span><span class="p">)</span>
+            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">datasets</span><span class="p">()</span><span class="o">.</span><span class="n">patch</span><span class="p">(</span><span class="n">projectId</span><span class="o">=</span><span class="n">source_project</span><span class="p">,</span>
+                                                 <span class="n">datasetId</span><span class="o">=</span><span class="n">source_dataset</span><span class="p">,</span>
+                                                 <span class="n">body</span><span class="o">=</span><span class="p">{</span><span class="s1">&#39;access&#39;</span><span class="p">:</span> <span class="n">access</span><span class="p">})</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="c1"># if view is already in access, do nothing.</span>
+            <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;table </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1"> already has authorized view access to </span><span class="si">%s</span><span class="s1">:</span><span class="si">%s</span><span class="s1"> dataset.&#39;</span><span class="p">,</span>
+                         <span class="n">view_project</span><span class="p">,</span> <span class="n">view_dataset</span><span class="p">,</span> <span class="n">view_table</span><span class="p">,</span>
+                         <span class="n">source_project</span><span class="p">,</span> <span class="n">source_dataset</span><span class="p">)</span>
+            <span class="k">return</span> <span class="n">source_dataset_resource</span>
+
+
+<span class="k">class</span> <span class="nc">BigQueryCursor</span><span class="p">(</span><span class="n">BigQueryBaseCursor</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    A very basic BigQuery PEP 249 cursor implementation. The PyHive PEP 249</span>
+<span class="sd">    implementation was used as a reference:</span>
+
+<span class="sd">    https://github.com/dropbox/PyHive/blob/master/pyhive/presto.py</span>
+<span class="sd">    https://github.com/dropbox/PyHive/blob/master/pyhive/common.py</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">service</span><span class="p">,</span> <span class="n">project_id</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">BigQueryCursor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="n">service</span><span class="o">=</span><span class="n">service</span><span class="p">,</span> <span class="n">project_id</span><span class="o">=</span><span class="n">project_id</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">buffersize</span> <span class="o">=</span> <span class="bp">None</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">page_token</span> <span class="o">=</span> <span class="bp">None</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">job_id</span> <span class="o">=</span> <span class="bp">None</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">buffer</span> <span class="o">=</span> <span class="p">[]</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">all_pages_loaded</span> <span class="o">=</span> <span class="bp">False</span>
+
+    <span class="nd">@property</span>
+    <span class="k">def</span> <span class="nf">description</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot; The schema description method is not currently implemented. &quot;&quot;&quot;</span>
+        <span class="k">raise</span> <span class="ne">NotImplementedError</span>
+
+    <span class="k">def</span> <span class="nf">close</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot; By default, do nothing &quot;&quot;&quot;</span>
+        <span class="k">pass</span>
+
+    <span class="nd">@property</span>
+    <span class="k">def</span> <span class="nf">rowcount</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot; By default, return -1 to indicate that this is not supported. &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="o">-</span><span class="mi">1</span>
+
+    <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">operation</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Executes a BigQuery query, and returns the job ID.</span>
+
+<span class="sd">        :param operation: The query to execute.</span>
+<span class="sd">        :type operation: string</span>
+<span class="sd">        :param parameters: Parameters to substitute into the query.</span>
+<span class="sd">        :type parameters: dict</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">bql</span> <span class="o">=</span> <span class="n">_bind_parameters</span><span class="p">(</span><span class="n">operation</span><span class="p">,</span> <span class="n">parameters</span><span class="p">)</span> <span class="k">if</span> <span class="n">parameters</span> <span class="k">else</span> <span class="n">operation</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">job_id</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_query</span><span class="p">(</span><span class="n">bql</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">executemany</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">operation</span><span class="p">,</span> <span class="n">seq_of_parameters</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Execute a BigQuery query multiple times with different parameters.</span>
+
+<span class="sd">        :param operation: The query to execute.</span>
+<span class="sd">        :type operation: string</span>
+<span class="sd">        :param parameters: List of dictionary parameters to substitute into the</span>
+<span class="sd">            query.</span>
+<span class="sd">        :type parameters: list</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">for</span> <span class="n">parameters</span> <span class="ow">in</span> <span class="n">seq_of_parameters</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">operation</span><span class="p">,</span> <span class="n">parameters</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">fetchone</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot; Fetch the next row of a query result set. &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">next</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">next</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Helper method for fetchone, which returns the next row from a buffer.</span>
+<span class="sd">        If the buffer is empty, attempts to paginate through the result set for</span>
+<span class="sd">        the next page, and load it into the buffer.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">job_id</span><span class="p">:</span>
+            <span class="k">return</span> <span class="bp">None</span>
+
+        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">buffer</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">all_pages_loaded</span><span class="p">:</span>
+                <span class="k">return</span> <span class="bp">None</span>
+
+            <span class="n">query_results</span> <span class="o">=</span> <span class="p">(</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">service</span><span class="o">.</span><span class="n">jobs</span><span class="p">()</span>
+                <span class="o">.</span><span class="n">getQueryResults</span><span class="p">(</span>
+                    <span class="n">projectId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
+                    <span class="n">jobId</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">job_id</span><span class="p">,</span>
+                    <span class="n">pageToken</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">page_token</span><span class="p">)</span>
+                <span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+            <span class="p">)</span>
+
+            <span class="k">if</span> <span class="s1">&#39;rows&#39;</span> <span class="ow">in</span> <span class="n">query_results</span> <span class="ow">and</span> <span class="n">query_results</span><span class="p">[</span><span class="s1">&#39;rows&#39;</span><span class="p">]:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">page_token</span> <span class="o">=</span> <span class="n">query_results</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;pageToken&#39;</span><span class="p">)</span>
+                <span class="n">fields</span> <span class="o">=</span> <span class="n">query_results</span><span class="p">[</span><span class="s1">&#39;schema&#39;</span><span class="p">][</span><span class="s1">&#39;fields&#39;</span><span class="p">]</span>
+                <span class="n">col_types</span> <span class="o">=</span> <span class="p">[</span><span class="n">field</span><span class="p">[</span><span class="s1">&#39;type&#39;</span><span class="p">]</span> <span class="k">for</span> <span class="n">field</span> <span class="ow">in</span> <span class="n">fields</span><span class="p">]</span>
+                <span class="n">rows</span> <span class="o">=</span> <span class="n">query_results</span><span class="p">[</span><span class="s1">&#39;rows&#39;</span><span class="p">]</span>
+
+                <span class="k">for</span> <span class="n">dict_row</span> <span class="ow">in</span> <span class="n">rows</span><span class="p">:</span>
+                    <span class="n">typed_row</span> <span class="o">=</span> <span class="p">([</span>
+                        <span class="n">_bq_cast</span><span class="p">(</span><span class="n">vs</span><span class="p">[</span><span class="s1">&#39;v&#39;</span><span class="p">],</span> <span class="n">col_types</span><span class="p">[</span><span class="n">idx</span><span class="p">])</span>
+                        <span class="k">for</span> <span class="n">idx</span><span class="p">,</span> <span class="n">vs</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">dict_row</span><span class="p">[</span><span class="s1">&#39;f&#39;</span><span class="p">])</span>
+                    <span class="p">])</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">buffer</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">typed_row</span><span class="p">)</span>
+
+                <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">page_token</span><span class="p">:</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">all_pages_loaded</span> <span class="o">=</span> <span class="bp">True</span>
+
+            <span class="k">else</span><span class="p">:</span>
+                <span class="c1"># Reset all state since we&#39;ve exhausted the results.</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">page_token</span> <span class="o">=</span> <span class="bp">None</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">job_id</span> <span class="o">=</span> <span class="bp">None</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">page_token</span> <span class="o">=</span> <span class="bp">None</span>
+                <span class="k">return</span> <span class="bp">None</span>
+
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">buffer</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">fetchmany</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">size</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Fetch the next set of rows of a query result, returning a sequence of sequences (e.g. a</span>
+<span class="sd">        list of tuples). An empty sequence is returned when no more rows are available.</span>
+<span class="sd">        The number of rows to fetch per call is specified by the parameter. If it is not given, the</span>
+<span class="sd">        cursor&#39;s arraysize determines the number of rows to be fetched. The method should try to</span>
+<span class="sd">        fetch as many rows as indicated by the size parameter. If this is not possible due to the</span>
+<span class="sd">        specified number of rows not being available, fewer rows may be returned.</span>
+<span cl

<TRUNCATED>

[3/6] incubator-airflow-site git commit: Adding incubator related disclaimer

[ma...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/502934ca/_modules/gcs_hook.html
----------------------------------------------------------------------
diff --git a/_modules/gcs_hook.html b/_modules/gcs_hook.html
index 012f4bf..c118b86 100644
--- a/_modules/gcs_hook.html
+++ b/_modules/gcs_hook.html
@@ -154,7 +154,7 @@
 
 <span class="kn">import</span> <span class="nn">logging</span>
 
-<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gc_base_hook</span> <span class="kn">import</span> <span class="n">GoogleCloudBaseHook</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcp_api_base_hook</span> <span class="kn">import</span> <span class="n">GoogleCloudBaseHook</span>
 <span class="kn">from</span> <span class="nn">apiclient.discovery</span> <span class="kn">import</span> <span class="n">build</span>
 <span class="kn">from</span> <span class="nn">apiclient.http</span> <span class="kn">import</span> <span class="n">MediaFileUpload</span>
 
@@ -163,19 +163,8 @@
 
 <div class="viewcode-block" id="GoogleCloudStorageHook"><a class="viewcode-back" href="../code.html#airflow.contrib.hooks.GoogleCloudStorageHook">[docs]</a><span class="k">class</span> <span class="nc">GoogleCloudStorageHook</span><span class="p">(</span><span class="n">GoogleCloudBaseHook</span><span class="p">):</span>
     <span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">    Interact with Google Cloud Storage. Connections must be defined with an</span>
-<span class="sd">    extras JSON field containing:</span>
-
-<span class="sd">    ::</span>
-<span class="sd">        {</span>
-<span class="sd">            &quot;project&quot;: &quot;&lt;google project ID&gt;&quot;,</span>
-<span class="sd">            &quot;service_account&quot;: &quot;&lt;google service account email&gt;&quot;,</span>
-<span class="sd">            &quot;key_path&quot;: &quot;&lt;p12 key path&gt;&quot;</span>
-<span class="sd">        }</span>
-
-<span class="sd">    If you have used ``gcloud auth`` to authenticate on the machine that&#39;s</span>
-<span class="sd">    running Airflow, you can exclude the service_account and key_path</span>
-<span class="sd">    parameters.</span>
+<span class="sd">    Interact with Google Cloud Storage. This hook uses the Google Cloud Platform</span>
+<span class="sd">    connection.</span>
 <span class="sd">    &quot;&quot;&quot;</span>
 
     <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/502934ca/_modules/index.html
----------------------------------------------------------------------
diff --git a/_modules/index.html b/_modules/index.html
index 7fe6be6..d40ab64 100644
--- a/_modules/index.html
+++ b/_modules/index.html
@@ -135,6 +135,11 @@
             
   <h1>All modules for which code is available</h1>
 <ul><li><a href="S3_hook.html">S3_hook</a></li>
+<li><a href="airflow/contrib/executors/mesos_executor.html">airflow.contrib.executors.mesos_executor</a></li>
+<li><a href="airflow/contrib/hooks/gcs_hook.html">airflow.contrib.hooks.gcs_hook</a></li>
+<li><a href="airflow/contrib/operators/bigquery_operator.html">airflow.contrib.operators.bigquery_operator</a></li>
+<li><a href="airflow/contrib/operators/bigquery_to_gcs.html">airflow.contrib.operators.bigquery_to_gcs</a></li>
+<li><a href="airflow/contrib/operators/gcs_download_operator.html">airflow.contrib.operators.gcs_download_operator</a></li>
 <li><a href="airflow/contrib/operators/hipchat_operator.html">airflow.contrib.operators.hipchat_operator</a></li>
 <li><a href="airflow/executors/celery_executor.html">airflow.executors.celery_executor</a></li>
 <li><a href="airflow/executors/local_executor.html">airflow.executors.local_executor</a></li>
@@ -145,6 +150,7 @@
 <li><a href="airflow/operators/docker_operator.html">airflow.operators.docker_operator</a></li>
 <li><a href="airflow/operators/sensors.html">airflow.operators.sensors</a></li>
 <li><a href="bash_operator.html">bash_operator</a></li>
+<li><a href="bigquery_hook.html">bigquery_hook</a></li>
 <li><a href="cloudant_hook.html">cloudant_hook</a></li>
 <li><a href="dagrun_operator.html">dagrun_operator</a></li>
 <li><a href="dbapi_hook.html">dbapi_hook</a></li>
@@ -172,6 +178,7 @@
 <li><a href="presto_check_operator.html">presto_check_operator</a></li>
 <li><a href="presto_hook.html">presto_hook</a></li>
 <li><a href="python_operator.html">python_operator</a></li>
+<li><a href="qubole_operator.html">qubole_operator</a></li>
 <li><a href="s3_to_hive_operator.html">s3_to_hive_operator</a></li>
 <li><a href="sensors.html">sensors</a></li>
 <li><a href="slack_operator.html">slack_operator</a></li>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/502934ca/_modules/qubole_operator.html
----------------------------------------------------------------------
diff --git a/_modules/qubole_operator.html b/_modules/qubole_operator.html
new file mode 100644
index 0000000..07ea626
--- /dev/null
+++ b/_modules/qubole_operator.html
@@ -0,0 +1,328 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>qubole_operator &mdash; Airflow Documentation</title>
+  
+
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+    <link rel="top" title="Airflow Documentation" href="../index.html"/>
+        <link rel="up" title="Module code" href="index.html"/> 
+
+  
+  <script src="../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+                <ul>
+<li class="toctree-l1"><a class="reference internal" href="../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+        <a href="../index.html">Airflow</a>
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+  <ul class="wy-breadcrumbs">
+    <li><a href="../index.html">Docs</a> &raquo;</li>
+      
+          <li><a href="index.html">Module code</a> &raquo;</li>
+      
+    <li>qubole_operator</li>
+      <li class="wy-breadcrumbs-aside">
+        
+          
+        
+      </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for qubole_operator</h1><div class="highlight"><pre>
+<span></span><span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks</span> <span class="kn">import</span> <span class="n">QuboleHook</span>
+
+
+<div class="viewcode-block" id="QuboleOperator"><a class="viewcode-back" href="../code.html#airflow.contrib.operators.QuboleOperator">[docs]</a><span class="k">class</span> <span class="nc">QuboleOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Execute tasks (commands) on QDS (https://qubole.com).</span>
+
+<span class="sd">    :param qubole_conn_id: Connection id which consists of qds auth_token</span>
+<span class="sd">    :type qubole_conn_id: str</span>
+
+<span class="sd">    kwargs:</span>
+<span class="sd">        :command_type: type of command to be executed, e.g. hivecmd, shellcmd, hadoopcmd</span>
+<span class="sd">        :tags: array of tags to be assigned with the command</span>
+<span class="sd">        :cluster_label: cluster label on which the command will be executed</span>
+<span class="sd">        :name: name to be given to command</span>
+
+<span class="sd">        **Arguments specific to command types**</span>
+
+<span class="sd">        hivecmd:</span>
+<span class="sd">            :query: inline query statement</span>
+<span class="sd">            :script_location: s3 location containing query statement</span>
+<span class="sd">            :sample_size: size of sample in bytes on which to run query</span>
+<span class="sd">            :macros: macro values which were used in query</span>
+<span class="sd">        prestocmd:</span>
+<span class="sd">            :query: inline query statement</span>
+<span class="sd">            :script_location: s3 location containing query statement</span>
+<span class="sd">            :macros: macro values which were used in query</span>
+<span class="sd">        hadoopcmd:</span>
+<span class="sd">            :sub_commnad: must be one these [&quot;jar&quot;, &quot;s3distcp&quot;, &quot;streaming&quot;] followed by 1 or more args</span>
+<span class="sd">        shellcmd:</span>
+<span class="sd">            :script: inline command with args</span>
+<span class="sd">            :script_location: s3 location containing query statement</span>
+<span class="sd">            :files: list of files in s3 bucket as file1,file2 format. These files will be copied into the working directory where the qubole command is being executed.</span>
+<span class="sd">            :archives: list of archives in s3 bucket as archive1,archive2 format. These will be unarchived intothe working directory where the qubole command is being executed</span>
+<span class="sd">            :parameters: any extra args which need to be passed to script (only when script_location is supplied)</span>
+<span class="sd">        pigcmd:</span>
+<span class="sd">            :script: inline query statement (latin_statements)</span>
+<span class="sd">            :script_location: s3 location containing pig query</span>
+<span class="sd">            :parameters: any extra args which need to be passed to script (only when script_location is supplied</span>
+<span class="sd">        sparkcmd:</span>
+<span class="sd">            :program: the complete Spark Program in Scala, SQL, Command, R, or Python</span>
+<span class="sd">            :cmdline: spark-submit command line, all required information must be specify in cmdline itself.</span>
+<span class="sd">            :sql: inline sql query</span>
+<span class="sd">            :script_location: s3 location containing query statement</span>
+<span class="sd">            :language: language of the program, Scala, SQL, Command, R, or Python</span>
+<span class="sd">            :app_id: ID of an Spark job server app</span>
+<span class="sd">            :arguments: spark-submit command line arguments</span>
+<span class="sd">            :user_program_arguments: arguments that the user program takes in</span>
+<span class="sd">            :macros: macro values which were used in query</span>
+<span class="sd">        dbtapquerycmd:</span>
+<span class="sd">            :db_tap_id: data store ID of the target database, in Qubole.</span>
+<span class="sd">            :query: inline query statement</span>
+<span class="sd">            :macros: macro values which were used in query</span>
+<span class="sd">        dbexportcmd:</span>
+<span class="sd">            :mode: 1 (simple), 2 (advance)</span>
+<span class="sd">            :hive_table: Name of the hive table</span>
+<span class="sd">            :partition_spec: partition specification for Hive table.</span>
+<span class="sd">            :dbtap_id: data store ID of the target database, in Qubole.</span>
+<span class="sd">            :db_table: name of the db table</span>
+<span class="sd">            :db_update_mode: allowinsert or updateonly</span>
+<span class="sd">            :db_update_keys: columns used to determine the uniqueness of rows</span>
+<span class="sd">            :export_dir: HDFS/S3 location from which data will be exported.</span>
+<span class="sd">            :fields_terminated_by: hex of the char used as column separator in the dataset.</span>
+<span class="sd">        dbimportcmd:</span>
+<span class="sd">            :mode: 1 (simple), 2 (advance)</span>
+<span class="sd">            :hive_table: Name of the hive table</span>
+<span class="sd">            :dbtap_id: data store ID of the target database, in Qubole.</span>
+<span class="sd">            :db_table: name of the db table</span>
+<span class="sd">            :where_clause: where clause, if any</span>
+<span class="sd">            :parallelism: number of parallel db connections to use for extracting data</span>
+<span class="sd">            :extract_query: SQL query to extract data from db. $CONDITIONS must be part of the where clause.</span>
+<span class="sd">            :boundary_query: Query to be used get range of row IDs to be extracted</span>
+<span class="sd">            :split_column: Column used as row ID to split data into ranges (mode 2)</span>
+
+<span class="sd">    .. note:: Following fields are template-supported : ``query``, ``script_location``, ``sub_command``, ``script``, ``files``,</span>
+<span class="sd">        ``archives``, ``program``, ``cmdline``, ``sql``, ``where_clause``, ``extract_query``, ``boundary_query``, ``macros``, ``tags``,</span>
+<span class="sd">        ``name``, ``parameters``. You can also use ``.txt`` files for template driven use cases.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;query&#39;</span><span class="p">,</span> <span class="s1">&#39;script_location&#39;</span><span class="p">,</span> <span class="s1">&#39;sub_command&#39;</span><span class="p">,</span> <span class="s1">&#39;script&#39;</span><span class="p">,</span> <span class="s1">&#39;files&#39;</span><span class="p">,</span> <span class="s1">&#39;archives&#39;</span><span class="p">,</span> <span class="s1">&#39;program&#39;</span><span class="p">,</span> <span class="s1">&#39;cmdline&#39;</span><span class="p">,</span>
+                       <span class="s1">&#39;sql&#39;</span><span class="p">,</span> <span class="s1">&#39;where_clause&#39;</span><span class="p">,</span> <span class="s1">&#39;extract_query&#39;</span><span class="p">,</span> <span class="s1">&#39;boundary_query&#39;</span><span class="p">,</span> <span class="s1">&#39;macros&#39;</span><span class="p">,</span> <span class="s1">&#39;tags&#39;</span><span class="p">,</span> <span class="s1">&#39;name&#39;</span><span class="p">,</span> <span class="s1">&#39;parameters&#39;</span><span class="p">)</span>
+    <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;.txt&#39;</span><span class="p">)</span>
+    <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#3064A1&#39;</span>
+    <span class="n">ui_fgcolor</span> <span class="o">=</span> <span class="s1">&#39;#fff&#39;</span>
+
+    <span class="nd">@apply_defaults</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">qubole_conn_id</span><span class="o">=</span><span class="s2">&quot;qubole_default&quot;</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">args</span> <span class="o">=</span> <span class="n">args</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span> <span class="o">=</span> <span class="n">kwargs</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span><span class="p">[</span><span class="s1">&#39;qubole_conn_id&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">qubole_conn_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">QuboleHook</span><span class="p">(</span><span class="o">*</span><span class="bp">self</span><span class="o">.</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span><span class="p">)</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">QuboleOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+        <span class="c1"># Reinitiating the hook, as some template fields might have changed</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">QuboleHook</span><span class="p">(</span><span class="o">*</span><span class="bp">self</span><span class="o">.</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span><span class="p">)</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">context</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">on_kill</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ti</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">kill</span><span class="p">(</span><span class="n">ti</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">get_results</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ti</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">fp</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">inline</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span> <span class="n">delim</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">fetch</span><span class="o">=</span><span class="bp">True</span><span class="p">):</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">get_results</span><span class="p">(</span><span class="n">ti</span><span class="p">,</span> <span class="n">fp</span><span class="p">,</span> <span class="n">inline</span><span class="p">,</span> <span class="n">delim</span><span class="p">,</span> <span class="n">fetch</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">get_log</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ti</span><span class="p">):</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">get_log</span><span class="p">(</span><span class="n">ti</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">get_jobs_id</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ti</span><span class="p">):</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">get_jobs_id</span><span class="p">(</span><span class="n">ti</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">__getattribute__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
+        <span class="k">if</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">QuboleOperator</span><span class="o">.</span><span class="n">template_fields</span><span class="p">:</span>
+            <span class="k">if</span> <span class="n">name</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span><span class="p">:</span>
+                <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span><span class="p">[</span><span class="n">name</span><span class="p">]</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="k">return</span> <span class="s1">&#39;&#39;</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="k">return</span> <span class="nb">object</span><span class="o">.</span><span class="n">__getattribute__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">__setattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
+        <span class="k">if</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">QuboleOperator</span><span class="o">.</span><span class="n">template_fields</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span><span class="p">[</span><span class="n">name</span><span class="p">]</span> <span class="o">=</span> <span class="n">value</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="nb">object</span><span class="o">.</span><span class="n">__setattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">value</span><span class="p">)</span></div>
+
+
+
+</pre></div>
+
+           </div>
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'../',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true
+        };
+    </script>
+      <script type="text/javascript" src="../_static/jquery.js"></script>
+      <script type="text/javascript" src="../_static/underscore.js"></script>
+      <script type="text/javascript" src="../_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="../_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/502934ca/_sources/code.txt
----------------------------------------------------------------------
diff --git a/_sources/code.txt b/_sources/code.txt
index 8a48b81..2876876 100644
--- a/_sources/code.txt
+++ b/_sources/code.txt
@@ -90,13 +90,13 @@ Community-contributed Operators
 .. automodule:: airflow.contrib.operators
     :show-inheritance:
     :members:
-        BigQueryOperator,
-        BigQueryToCloudStorageOperator,
-        GoogleCloudStorageDownloadOperator,
         SSHExecuteOperator,
         VerticaOperator,
         VerticaToHiveTransfer
 
+.. autoclass:: airflow.contrib.operators.bigquery_operator.BigQueryOperator
+.. autoclass:: airflow.contrib.operators.bigquery_to_gcs.BigQueryToCloudStorageOperator
+.. autoclass:: airflow.contrib.operators.gcs_download_operator.GoogleCloudStorageDownloadOperator
 .. autoclass:: airflow.contrib.operators.QuboleOperator
 .. autoclass:: airflow.contrib.operators.hipchat_operator.HipChatAPIOperator
 .. autoclass:: airflow.contrib.operators.hipchat_operator.HipChatAPISendRoomNotificationOperator
@@ -226,6 +226,8 @@ Community contributed hooks
         SSHHook,
         CloudantHook
 
+.. autoclass:: airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook
+
 Executors
 ---------
 Executors are the mechanism by which task instances get run.
@@ -237,7 +239,4 @@ Executors are the mechanism by which task instances get run.
 Community-contributed executors
 '''''''''''''''''''''''''''''''
 
-.. automodule:: airflow.contrib.executors
-    :show-inheritance:
-    :members:
-        MesosExecutor
+.. autoclass:: airflow.contrib.executors.mesos_executor.MesosExecutor

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/502934ca/_sources/concepts.txt
----------------------------------------------------------------------
diff --git a/_sources/concepts.txt b/_sources/concepts.txt
index 6e15ff8..811130d 100644
--- a/_sources/concepts.txt
+++ b/_sources/concepts.txt
@@ -664,6 +664,7 @@ configuration files, it allows you to expose the configuration that led
 to the related tasks in Airflow.
 
 .. code:: python
+
     """
     ### My great DAG
     """

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/502934ca/_sources/index.txt
----------------------------------------------------------------------
diff --git a/_sources/index.txt b/_sources/index.txt
index 70f9355..c98d619 100644
--- a/_sources/index.txt
+++ b/_sources/index.txt
@@ -5,7 +5,19 @@
     :width: 150
 
 Apache Airflow (incubating) Documentation
-================================
+=========================================
+
+.. note ::
+
+    Apache Airflow is an effort undergoing incubation at The Apache
+    Software Foundation (ASF), sponsored by Chris Riccomini.
+    Incubation is required of all newly accepted projects until a further
+    review indicates that the infrastructure, communications, and
+    decision making process have stabilized in a manner consistent with
+    other successful ASF projects. While incubation status is not
+    necessarily a reflection of the completeness or stability of
+    the code, it does indicate that the project has yet to be fully
+    endorsed by the ASF.
 
 Airflow is a platform to programmatically author, schedule and monitor
 workflows.
@@ -20,7 +32,6 @@ monitor progress, and troubleshoot issues when needed.
 When workflows are defined as code, they become more maintainable,
 versionable, testable, and collaborative.
 
-------------
 
 
 .. image:: img/airflow.gif

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/502934ca/cli.html
----------------------------------------------------------------------
diff --git a/cli.html b/cli.html
index c19ab2f..e8785c4 100644
--- a/cli.html
+++ b/cli.html
@@ -143,7 +143,7 @@
 many types of operation on a DAG, starting services, and supporting
 development and testing.</p>
 <div class="highlight-default"><div class="highlight"><pre><span></span><span class="n">usage</span><span class="p">:</span> <span class="n">airflow</span> <span class="p">[</span><span class="o">-</span><span class="n">h</span><span class="p">]</span>
-               <span class="p">{</span><span class="n">resetdb</span><span class="p">,</span><span class="n">render</span><span class="p">,</span><span class="n">variables</span><span class="p">,</span><span class="n">pause</span><span class="p">,</span><span class="n">version</span><span class="p">,</span><span class="n">initdb</span><span class="p">,</span><span class="n">test</span><span class="p">,</span><span class="n">unpause</span><span class="p">,</span><span class="n">run</span><span class="p">,</span><span class="n">list_tasks</span><span class="p">,</span><span class="n">backfill</span><span class="p">,</span><span class="n">list_dags</span><span class="p">,</span><span class="n">kerberos</span><span class="p">,</span><span class="n">worker</span><span class="p">,</span><span class="n">webserver</span><span class="p">,</span><span class="n">flower</span><span class="p">,</span><span class="n">scheduler</span><span class="p">,</span><span class="n">task_state</span><span c
 lass="p">,</span><span class="n">trigger_dag</span><span class="p">,</span><span class="n">serve_logs</span><span class="p">,</span><span class="n">clear</span><span class="p">,</span><span class="n">upgradedb</span><span class="p">}</span>
+               <span class="p">{</span><span class="n">resetdb</span><span class="p">,</span><span class="n">render</span><span class="p">,</span><span class="n">variables</span><span class="p">,</span><span class="n">pause</span><span class="p">,</span><span class="n">version</span><span class="p">,</span><span class="n">initdb</span><span class="p">,</span><span class="n">test</span><span class="p">,</span><span class="n">unpause</span><span class="p">,</span><span class="n">dag_state</span><span class="p">,</span><span class="n">run</span><span class="p">,</span><span class="n">list_tasks</span><span class="p">,</span><span class="n">backfill</span><span class="p">,</span><span class="n">list_dags</span><span class="p">,</span><span class="n">kerberos</span><span class="p">,</span><span class="n">worker</span><span class="p">,</span><span class="n">webserver</span><span class="p">,</span><span class="n">flower</span><span class="p">,</span><span class="n">scheduler</span><span cl
 ass="p">,</span><span class="n">task_state</span><span class="p">,</span><span class="n">trigger_dag</span><span class="p">,</span><span class="n">serve_logs</span><span class="p">,</span><span class="n">clear</span><span class="p">,</span><span class="n">upgradedb</span><span class="p">}</span>
                <span class="o">...</span>
 </pre></div>
 </div>
@@ -330,7 +330,7 @@ development and testing.</p>
 </dl>
 </dd>
 <dt><strong>unpause</strong></dt>
-<dd><p class="first">Pause a DAG</p>
+<dd><p class="first">Resume a paused DAG</p>
 <div class="highlight-default"><div class="highlight"><pre><span></span><span class="n">usage</span><span class="p">:</span> <span class="n">airflow</span> <span class="n">unpause</span> <span class="p">[</span><span class="o">-</span><span class="n">h</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">sd</span> <span class="n">SUBDIR</span><span class="p">]</span> <span class="n">dag_id</span>
 </pre></div>
 </div>
@@ -360,6 +360,40 @@ development and testing.</p>
 </dd>
 </dl>
 </dd>
+<dt><strong>dag_state</strong></dt>
+<dd><p class="first">Get the status of a dag run</p>
+<div class="highlight-default"><div class="highlight"><pre><span></span><span class="n">usage</span><span class="p">:</span> <span class="n">airflow</span> <span class="n">dag_state</span> <span class="p">[</span><span class="o">-</span><span class="n">h</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">sd</span> <span class="n">SUBDIR</span><span class="p">]</span> <span class="n">dag_id</span> <span class="n">execution_date</span>
+</pre></div>
+</div>
+<dl class="last docutils">
+<dt>Positional arguments:</dt>
+<dd><table class="first last docutils option-list" frame="void" rules="none">
+<col class="option" />
+<col class="description" />
+<tbody valign="top">
+<tr><td class="option-group">
+<kbd>dag_id</kbd></td>
+<td>The id of the dag</td></tr>
+<tr><td class="option-group">
+<kbd>execution_date</kbd></td>
+<td>The execution date of the DAG</td></tr>
+</tbody>
+</table>
+</dd>
+<dt>Options:</dt>
+<dd><table class="first last docutils option-list" frame="void" rules="none">
+<col class="option" />
+<col class="description" />
+<tbody valign="top">
+<tr><td class="option-group" colspan="2">
+<kbd><span class="option">-sd=/Users/maxime_beauchemin/dags</span>, <span class="option">--subdir=/Users/maxime_beauchemin/dags</span></kbd></td>
+</tr>
+<tr><td>&nbsp;</td><td>File location or directory from which to look for the dag</td></tr>
+</tbody>
+</table>
+</dd>
+</dl>
+</dd>
 <dt><strong>run</strong></dt>
 <dd><p class="first">Run a single task instance</p>
 <div class="highlight-default"><div class="highlight"><pre><span></span><span class="n">usage</span><span class="p">:</span> <span class="n">airflow</span> <span class="n">run</span> <span class="p">[</span><span class="o">-</span><span class="n">h</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">sd</span> <span class="n">SUBDIR</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">m</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">f</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">pool</span> <span class="n">POOL</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">l</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">i</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">I</span>
 <span class="p">]</span>
@@ -401,7 +435,7 @@ development and testing.</p>
 <tr><td class="option-group" colspan="2">
 <kbd><span class="option">-f=False</span>, <span class="option">--force=False</span></kbd></td>
 </tr>
-<tr><td>&nbsp;</td><td>Force a run regardless or previous success</td></tr>
+<tr><td>&nbsp;</td><td>Force a run regardless of previous success</td></tr>
 <tr><td class="option-group">
 <kbd><span class="option">--pool</span></kbd></td>
 <td>Resource pool to use</td></tr>
@@ -542,7 +576,7 @@ development and testing.</p>
 </dd>
 <dt><strong>list_dags</strong></dt>
 <dd><p class="first">List all the DAGs</p>
-<div class="highlight-default"><div class="highlight"><pre><span></span><span class="n">usage</span><span class="p">:</span> <span class="n">airflow</span> <span class="n">list_dags</span> <span class="p">[</span><span class="o">-</span><span class="n">h</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">sd</span> <span class="n">SUBDIR</span><span class="p">]</span>
+<div class="highlight-default"><div class="highlight"><pre><span></span><span class="n">usage</span><span class="p">:</span> <span class="n">airflow</span> <span class="n">list_dags</span> <span class="p">[</span><span class="o">-</span><span class="n">h</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">sd</span> <span class="n">SUBDIR</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">r</span><span class="p">]</span>
 </pre></div>
 </div>
 <dl class="last docutils">
@@ -555,6 +589,10 @@ development and testing.</p>
 <kbd><span class="option">-sd=/Users/maxime_beauchemin/dags</span>, <span class="option">--subdir=/Users/maxime_beauchemin/dags</span></kbd></td>
 </tr>
 <tr><td>&nbsp;</td><td>File location or directory from which to look for the dag</td></tr>
+<tr><td class="option-group" colspan="2">
+<kbd><span class="option">-r=False</span>, <span class="option">--report=False</span></kbd></td>
+</tr>
+<tr><td>&nbsp;</td><td>Show DagBag loading report</td></tr>
 </tbody>
 </table>
 </dd>
@@ -594,7 +632,7 @@ development and testing.</p>
 <tr><td class="option-group" colspan="2">
 <kbd><span class="option">-D=False</span>, <span class="option">--daemon=False</span></kbd></td>
 </tr>
-<tr><td>&nbsp;</td><td>Daemonize instead of running on the foreground</td></tr>
+<tr><td>&nbsp;</td><td>Daemonize instead of running in the foreground</td></tr>
 <tr><td class="option-group">
 <kbd><span class="option">--stdout</span></kbd></td>
 <td>Redirect stdout to this file</td></tr>
@@ -639,7 +677,7 @@ development and testing.</p>
 <tr><td class="option-group" colspan="2">
 <kbd><span class="option">-D=False</span>, <span class="option">--daemon=False</span></kbd></td>
 </tr>
-<tr><td>&nbsp;</td><td>Daemonize instead of running on the foreground</td></tr>
+<tr><td>&nbsp;</td><td>Daemonize instead of running in the foreground</td></tr>
 <tr><td class="option-group">
 <kbd><span class="option">--stdout</span></kbd></td>
 <td>Redirect stdout to this file</td></tr>
@@ -659,7 +697,8 @@ development and testing.</p>
 <div class="highlight-default"><div class="highlight"><pre><span></span><span class="n">usage</span><span class="p">:</span> <span class="n">airflow</span> <span class="n">webserver</span> <span class="p">[</span><span class="o">-</span><span class="n">h</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">p</span> <span class="n">PORT</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">w</span> <span class="n">WORKERS</span><span class="p">]</span>
                          <span class="p">[</span><span class="o">-</span><span class="n">k</span> <span class="p">{</span><span class="n">sync</span><span class="p">,</span><span class="n">eventlet</span><span class="p">,</span><span class="n">gevent</span><span class="p">,</span><span class="n">tornado</span><span class="p">}]</span>
                          <span class="p">[</span><span class="o">-</span><span class="n">t</span> <span class="n">WORKER_TIMEOUT</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">hn</span> <span class="n">HOSTNAME</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">pid</span> <span class="p">[</span><span class="n">PID</span><span class="p">]]</span> <span class="p">[</span><span class="o">-</span><span class="n">D</span><span class="p">]</span>
-                         <span class="p">[</span><span class="o">--</span><span class="n">stdout</span> <span class="n">STDOUT</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">stderr</span> <span class="n">STDERR</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">l</span> <span class="n">LOG_FILE</span><span class="p">]</span>
+                         <span class="p">[</span><span class="o">--</span><span class="n">stdout</span> <span class="n">STDOUT</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">stderr</span> <span class="n">STDERR</span><span class="p">]</span>
+                         <span class="p">[</span><span class="o">-</span><span class="n">A</span> <span class="n">ACCESS_LOGFILE</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">E</span> <span class="n">ERROR_LOGFILE</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">l</span> <span class="n">LOG_FILE</span><span class="p">]</span>
                          <span class="p">[</span><span class="o">-</span><span class="n">d</span><span class="p">]</span>
 </pre></div>
 </div>
@@ -680,7 +719,7 @@ development and testing.</p>
 <tr><td class="option-group" colspan="2">
 <kbd><span class="option">-k=sync</span>, <span class="option">--workerclass=sync</span></kbd></td>
 </tr>
-<tr><td>&nbsp;</td><td><p class="first">The worker class to use for gunicorn</p>
+<tr><td>&nbsp;</td><td><p class="first">The worker class to use for Gunicorn</p>
 <p class="last">Possible choices: sync, eventlet, gevent, tornado</p>
 </td></tr>
 <tr><td class="option-group" colspan="2">
@@ -697,13 +736,21 @@ development and testing.</p>
 <tr><td class="option-group" colspan="2">
 <kbd><span class="option">-D=False</span>, <span class="option">--daemon=False</span></kbd></td>
 </tr>
-<tr><td>&nbsp;</td><td>Daemonize instead of running on the foreground</td></tr>
+<tr><td>&nbsp;</td><td>Daemonize instead of running in the foreground</td></tr>
 <tr><td class="option-group">
 <kbd><span class="option">--stdout</span></kbd></td>
 <td>Redirect stdout to this file</td></tr>
 <tr><td class="option-group">
 <kbd><span class="option">--stderr</span></kbd></td>
 <td>Redirect stderr to this file</td></tr>
+<tr><td class="option-group" colspan="2">
+<kbd><span class="option">-A</span>, <span class="option">--access_logfile</span></kbd></td>
+</tr>
+<tr><td>&nbsp;</td><td>The logfile to store the webserver access log. Use &#8216;-&#8216; to print to stderr.</td></tr>
+<tr><td class="option-group" colspan="2">
+<kbd><span class="option">-E</span>, <span class="option">--error_logfile</span></kbd></td>
+</tr>
+<tr><td>&nbsp;</td><td>The logfile to store the webserver error log. Use &#8216;-&#8216; to print to stderr.</td></tr>
 <tr><td class="option-group">
 <kbd><span class="option">-l</span>, <span class="option">--log-file</span></kbd></td>
 <td>Location of the log file</td></tr>
@@ -742,7 +789,7 @@ development and testing.</p>
 <tr><td class="option-group" colspan="2">
 <kbd><span class="option">-D=False</span>, <span class="option">--daemon=False</span></kbd></td>
 </tr>
-<tr><td>&nbsp;</td><td>Daemonize instead of running on the foreground</td></tr>
+<tr><td>&nbsp;</td><td>Daemonize instead of running in the foreground</td></tr>
 <tr><td class="option-group">
 <kbd><span class="option">--stdout</span></kbd></td>
 <td>Redirect stdout to this file</td></tr>
@@ -758,7 +805,7 @@ development and testing.</p>
 </dl>
 </dd>
 <dt><strong>scheduler</strong></dt>
-<dd><p class="first">Start a scheduler scheduler instance</p>
+<dd><p class="first">Start a scheduler instance</p>
 <div class="highlight-default"><div class="highlight"><pre><span></span><span class="n">usage</span><span class="p">:</span> <span class="n">airflow</span> <span class="n">scheduler</span> <span class="p">[</span><span class="o">-</span><span class="n">h</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">d</span> <span class="n">DAG_ID</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">sd</span> <span class="n">SUBDIR</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">n</span> <span class="n">NUM_RUNS</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">p</span><span class="p">]</span>
                          <span class="p">[</span><span class="o">--</span><span class="n">pid</span> <span class="p">[</span><span class="n">PID</span><span class="p">]]</span> <span class="p">[</span><span class="o">-</span><span class="n">D</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">stdout</span> <span class="n">STDOUT</span><span class="p">]</span>
                          <span class="p">[</span><span class="o">--</span><span class="n">stderr</span> <span class="n">STDERR</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">l</span> <span class="n">LOG_FILE</span><span class="p">]</span>
@@ -790,7 +837,7 @@ development and testing.</p>
 <tr><td class="option-group" colspan="2">
 <kbd><span class="option">-D=False</span>, <span class="option">--daemon=False</span></kbd></td>
 </tr>
-<tr><td>&nbsp;</td><td>Daemonize instead of running on the foreground</td></tr>
+<tr><td>&nbsp;</td><td>Daemonize instead of running in the foreground</td></tr>
 <tr><td class="option-group">
 <kbd><span class="option">--stdout</span></kbd></td>
 <td>Redirect stdout to this file</td></tr>
@@ -870,10 +917,10 @@ development and testing.</p>
 <tr><td>&nbsp;</td><td>File location or directory from which to look for the dag</td></tr>
 <tr><td class="option-group">
 <kbd><span class="option">-r</span>, <span class="option">--run_id</span></kbd></td>
-<td>Helps to indentify this run</td></tr>
+<td>Helps to identify this run</td></tr>
 <tr><td class="option-group">
 <kbd><span class="option">-c</span>, <span class="option">--conf</span></kbd></td>
-<td>json string that gets pickled into the DagRun&#8217;s conf attribute</td></tr>
+<td>JSON string that gets pickled into the DagRun&#8217;s conf attribute</td></tr>
 </tbody>
 </table>
 </dd>
@@ -890,7 +937,7 @@ development and testing.</p>
 <dt><strong>clear</strong></dt>
 <dd><p class="first">Clear a set of task instance, as if they never ran</p>
 <div class="highlight-default"><div class="highlight"><pre><span></span><span class="n">usage</span><span class="p">:</span> <span class="n">airflow</span> <span class="n">clear</span> <span class="p">[</span><span class="o">-</span><span class="n">h</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">t</span> <span class="n">TASK_REGEX</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">s</span> <span class="n">START_DATE</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">e</span> <span class="n">END_DATE</span><span class="p">]</span>
-                     <span class="p">[</span><span class="o">-</span><span class="n">sd</span> <span class="n">SUBDIR</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">u</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">d</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">c</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">f</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">r</span><span class="p">]</span>
+                     <span class="p">[</span><span class="o">-</span><span class="n">sd</span> <span class="n">SUBDIR</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">u</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">d</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">c</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">f</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">r</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">x</span><span class="p">]</span>
                      <span class="n">dag_id</span>
 </pre></div>
 </div>
@@ -946,13 +993,17 @@ development and testing.</p>
 <kbd><span class="option">-r=False</span>, <span class="option">--only_running=False</span></kbd></td>
 </tr>
 <tr><td>&nbsp;</td><td>Only running jobs</td></tr>
+<tr><td class="option-group" colspan="2">
+<kbd><span class="option">-x=False</span>, <span class="option">--exclude_subdags=False</span></kbd></td>
+</tr>
+<tr><td>&nbsp;</td><td>Exclude subdags</td></tr>
 </tbody>
 </table>
 </dd>
 </dl>
 </dd>
 <dt><strong>upgradedb</strong></dt>
-<dd><p class="first">Upgrade metadata database to latest version</p>
+<dd><p class="first">Upgrade the metadata database to latest version</p>
 <div class="highlight-default"><div class="highlight"><pre><span></span><span class="n">usage</span><span class="p">:</span> <span class="n">airflow</span> <span class="n">upgradedb</span> <span class="p">[</span><span class="o">-</span><span class="n">h</span><span class="p">]</span>
 </pre></div>
 </div>

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/502934ca/code.html
----------------------------------------------------------------------
diff --git a/code.html b/code.html
index 3bb26b1..06a0aab 100644
--- a/code.html
+++ b/code.html
@@ -96,7 +96,7 @@
 <li class="toctree-l3"><a class="reference internal" href="#baseoperator">BaseOperator</a></li>
 <li class="toctree-l3"><a class="reference internal" href="#basesensoroperator">BaseSensorOperator</a></li>
 <li class="toctree-l3"><a class="reference internal" href="#module-airflow.operators">Operator API</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#community-contributed-operators">Community-contributed Operators</a></li>
+<li class="toctree-l3"><a class="reference internal" href="#module-airflow.contrib.operators">Community-contributed Operators</a></li>
 </ul>
 </li>
 <li class="toctree-l2"><a class="reference internal" href="#macros">Macros</a><ul>
@@ -106,7 +106,7 @@
 </li>
 <li class="toctree-l2"><a class="reference internal" href="#models">Models</a></li>
 <li class="toctree-l2"><a class="reference internal" href="#module-airflow.hooks">Hooks</a><ul>
-<li class="toctree-l3"><a class="reference internal" href="#community-contributed-hooks">Community contributed hooks</a></li>
+<li class="toctree-l3"><a class="reference internal" href="#module-airflow.contrib.hooks">Community contributed hooks</a></li>
 </ul>
 </li>
 <li class="toctree-l2"><a class="reference internal" href="#executors">Executors</a><ul>
@@ -1186,9 +1186,9 @@ The default is False.</li>
 </dd></dl>
 
 </div>
-<div class="section" id="community-contributed-operators">
-<h3>Community-contributed Operators<a class="headerlink" href="#community-contributed-operators" title="Permalink to this headline">�</a></h3>
-<span class="target" id="module-airflow.contrib.operators"></span><dl class="class">
+<div class="section" id="module-airflow.contrib.operators">
+<span id="community-contributed-operators"></span><h3>Community-contributed Operators<a class="headerlink" href="#module-airflow.contrib.operators" title="Permalink to this headline">�</a></h3>
+<dl class="class">
 <dt id="airflow.contrib.operators.SSHExecuteOperator">
 <em class="property">class </em><code class="descclassname">airflow.contrib.operators.</code><code class="descname">SSHExecuteOperator</code><span class="sig-paren">(</span><em>ssh_hook</em>, <em>bash_command</em>, <em>xcom_push=False</em>, <em>env=None</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/ssh_execute_operator.html#SSHExecuteOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.SSHExecuteOperator" title="Permalink to this definition">�</a></dt>
 <dd><p>Bases: <a class="reference internal" href="#airflow.models.BaseOperator" title="airflow.models.BaseOperator"><code class="xref py py-class docutils literal"><span class="pre">airflow.models.BaseOperator</span></code></a></p>
@@ -1273,6 +1273,236 @@ specific database</li>
 </dd></dl>
 
 <dl class="class">
+<dt id="airflow.contrib.operators.bigquery_operator.BigQueryOperator">
+<em class="property">class </em><code class="descclassname">airflow.contrib.operators.bigquery_operator.</code><code class="descname">BigQueryOperator</code><span class="sig-paren">(</span><em>bql</em>, <em>destination_dataset_table=False</em>, <em>write_disposition='WRITE_EMPTY'</em>, <em>allow_large_results=False</em>, <em>bigquery_conn_id='bigquery_default'</em>, <em>delegate_to=None</em>, <em>udf_config=False</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/contrib/operators/bigquery_operator.html#BigQueryOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.bigquery_operator.BigQueryOperator" title="Permalink to this definition">�</a></dt>
+<dd><p>Executes BigQuery SQL queries in a specific BigQuery database</p>
+</dd></dl>
+
+<dl class="class">
+<dt id="airflow.contrib.operators.bigquery_to_gcs.BigQueryToCloudStorageOperator">
+<em class="property">class </em><code class="descclassname">airflow.contrib.operators.bigquery_to_gcs.</code><code class="descname">BigQueryToCloudStorageOperator</code><span class="sig-paren">(</span><em>source_project_dataset_table</em>, <em>destination_cloud_storage_uris</em>, <em>compression='NONE'</em>, <em>export_format='CSV'</em>, <em>field_delimiter='</em>, <em>'</em>, <em>print_header=True</em>, <em>bigquery_conn_id='bigquery_default'</em>, <em>delegate_to=None</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/contrib/operators/bigquery_to_gcs.html#BigQueryToCloudStorageOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.bigquery_to_gcs.BigQueryToCloudStorageOperator" title="Permalink to this definition">�</a></dt>
+<dd><p>Transfers a BigQuery table to a Google Cloud Storage bucket.</p>
+</dd></dl>
+
+<dl class="class">
+<dt id="airflow.contrib.operators.gcs_download_operator.GoogleCloudStorageDownloadOperator">
+<em class="property">class </em><code class="descclassname">airflow.contrib.operators.gcs_download_operator.</code><code class="descname">GoogleCloudStorageDownloadOperator</code><span class="sig-paren">(</span><em>bucket</em>, <em>object</em>, <em>filename</em>, <em>google_cloud_storage_conn_id='google_cloud_storage_default'</em>, <em>delegate_to=None</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/contrib/operators/gcs_download_operator.html#GoogleCloudStorageDownloadOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.gcs_download_operator.GoogleCloudStorageDownloadOperator" title="Permalink to this definition">�</a></dt>
+<dd><p>Downloads a file from Google Cloud Storage.</p>
+</dd></dl>
+
+<dl class="class">
+<dt id="airflow.contrib.operators.QuboleOperator">
+<em class="property">class </em><code class="descclassname">airflow.contrib.operators.</code><code class="descname">QuboleOperator</code><span class="sig-paren">(</span><em>qubole_conn_id='qubole_default'</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/qubole_operator.html#QuboleOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.QuboleOperator" title="Permalink to this definition">�</a></dt>
+<dd><p>Execute tasks (commands) on QDS (<a class="reference external" href="https://qubole.com">https://qubole.com</a>).</p>
+<table class="docutils field-list" frame="void" rules="none">
+<col class="field-name" />
+<col class="field-body" />
+<tbody valign="top">
+<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>qubole_conn_id</strong> (<em>str</em>) &#8211; Connection id which consists of qds auth_token</td>
+</tr>
+</tbody>
+</table>
+<dl class="docutils">
+<dt>kwargs:</dt>
+<dd><table class="first docutils field-list" frame="void" rules="none">
+<col class="field-name" />
+<col class="field-body" />
+<tbody valign="top">
+<tr class="field-odd field"><th class="field-name">command_type:</th><td class="field-body">type of command to be executed, e.g. hivecmd, shellcmd, hadoopcmd</td>
+</tr>
+<tr class="field-even field"><th class="field-name">tags:</th><td class="field-body">array of tags to be assigned with the command</td>
+</tr>
+<tr class="field-odd field"><th class="field-name">cluster_label:</th><td class="field-body">cluster label on which the command will be executed</td>
+</tr>
+<tr class="field-even field"><th class="field-name">name:</th><td class="field-body">name to be given to command</td>
+</tr>
+</tbody>
+</table>
+<p><strong>Arguments specific to command types</strong></p>
+<dl class="last docutils">
+<dt>hivecmd:</dt>
+<dd><table class="first last docutils field-list" frame="void" rules="none">
+<col class="field-name" />
+<col class="field-body" />
+<tbody valign="top">
+<tr class="field-odd field"><th class="field-name">query:</th><td class="field-body">inline query statement</td>
+</tr>
+<tr class="field-even field"><th class="field-name" colspan="2">script_location:</th></tr>
+<tr class="field-even field"><td>&nbsp;</td><td class="field-body">s3 location containing query statement</td>
+</tr>
+<tr class="field-odd field"><th class="field-name">sample_size:</th><td class="field-body">size of sample in bytes on which to run query</td>
+</tr>
+<tr class="field-even field"><th class="field-name">macros:</th><td class="field-body">macro values which were used in query</td>
+</tr>
+</tbody>
+</table>
+</dd>
+<dt>prestocmd:</dt>
+<dd><table class="first last docutils field-list" frame="void" rules="none">
+<col class="field-name" />
+<col class="field-body" />
+<tbody valign="top">
+<tr class="field-odd field"><th class="field-name">query:</th><td class="field-body">inline query statement</td>
+</tr>
+<tr class="field-even field"><th class="field-name" colspan="2">script_location:</th></tr>
+<tr class="field-even field"><td>&nbsp;</td><td class="field-body">s3 location containing query statement</td>
+</tr>
+<tr class="field-odd field"><th class="field-name">macros:</th><td class="field-body">macro values which were used in query</td>
+</tr>
+</tbody>
+</table>
+</dd>
+<dt>hadoopcmd:</dt>
+<dd><table class="first last docutils field-list" frame="void" rules="none">
+<col class="field-name" />
+<col class="field-body" />
+<tbody valign="top">
+<tr class="field-odd field"><th class="field-name">sub_commnad:</th><td class="field-body">must be one these [&#8220;jar&#8221;, &#8220;s3distcp&#8221;, &#8220;streaming&#8221;] followed by 1 or more args</td>
+</tr>
+</tbody>
+</table>
+</dd>
+<dt>shellcmd:</dt>
+<dd><table class="first last docutils field-list" frame="void" rules="none">
+<col class="field-name" />
+<col class="field-body" />
+<tbody valign="top">
+<tr class="field-odd field"><th class="field-name">script:</th><td class="field-body">inline command with args</td>
+</tr>
+<tr class="field-even field"><th class="field-name" colspan="2">script_location:</th></tr>
+<tr class="field-even field"><td>&nbsp;</td><td class="field-body">s3 location containing query statement</td>
+</tr>
+<tr class="field-odd field"><th class="field-name">files:</th><td class="field-body">list of files in s3 bucket as file1,file2 format. These files will be copied into the working directory where the qubole command is being executed.</td>
+</tr>
+<tr class="field-even field"><th class="field-name">archives:</th><td class="field-body">list of archives in s3 bucket as archive1,archive2 format. These will be unarchived intothe working directory where the qubole command is being executed</td>
+</tr>
+<tr class="field-odd field"><th class="field-name">parameters:</th><td class="field-body">any extra args which need to be passed to script (only when script_location is supplied)</td>
+</tr>
+</tbody>
+</table>
+</dd>
+<dt>pigcmd:</dt>
+<dd><table class="first last docutils field-list" frame="void" rules="none">
+<col class="field-name" />
+<col class="field-body" />
+<tbody valign="top">
+<tr class="field-odd field"><th class="field-name">script:</th><td class="field-body">inline query statement (latin_statements)</td>
+</tr>
+<tr class="field-even field"><th class="field-name" colspan="2">script_location:</th></tr>
+<tr class="field-even field"><td>&nbsp;</td><td class="field-body">s3 location containing pig query</td>
+</tr>
+<tr class="field-odd field"><th class="field-name">parameters:</th><td class="field-body">any extra args which need to be passed to script (only when script_location is supplied</td>
+</tr>
+</tbody>
+</table>
+</dd>
+<dt>sparkcmd:</dt>
+<dd><table class="first last docutils field-list" frame="void" rules="none">
+<col class="field-name" />
+<col class="field-body" />
+<tbody valign="top">
+<tr class="field-odd field"><th class="field-name">program:</th><td class="field-body">the complete Spark Program in Scala, SQL, Command, R, or Python</td>
+</tr>
+<tr class="field-even field"><th class="field-name">cmdline:</th><td class="field-body">spark-submit command line, all required information must be specify in cmdline itself.</td>
+</tr>
+<tr class="field-odd field"><th class="field-name">sql:</th><td class="field-body">inline sql query</td>
+</tr>
+<tr class="field-even field"><th class="field-name" colspan="2">script_location:</th></tr>
+<tr class="field-even field"><td>&nbsp;</td><td class="field-body">s3 location containing query statement</td>
+</tr>
+<tr class="field-odd field"><th class="field-name">language:</th><td class="field-body">language of the program, Scala, SQL, Command, R, or Python</td>
+</tr>
+<tr class="field-even field"><th class="field-name">app_id:</th><td class="field-body">ID of an Spark job server app</td>
+</tr>
+<tr class="field-odd field"><th class="field-name">arguments:</th><td class="field-body">spark-submit command line arguments</td>
+</tr>
+<tr class="field-even field"><th class="field-name" colspan="2">user_program_arguments:</th></tr>
+<tr class="field-even field"><td>&nbsp;</td><td class="field-body">arguments that the user program takes in</td>
+</tr>
+<tr class="field-odd field"><th class="field-name">macros:</th><td class="field-body">macro values which were used in query</td>
+</tr>
+</tbody>
+</table>
+</dd>
+<dt>dbtapquerycmd:</dt>
+<dd><table class="first last docutils field-list" frame="void" rules="none">
+<col class="field-name" />
+<col class="field-body" />
+<tbody valign="top">
+<tr class="field-odd field"><th class="field-name">db_tap_id:</th><td class="field-body">data store ID of the target database, in Qubole.</td>
+</tr>
+<tr class="field-even field"><th class="field-name">query:</th><td class="field-body">inline query statement</td>
+</tr>
+<tr class="field-odd field"><th class="field-name">macros:</th><td class="field-body">macro values which were used in query</td>
+</tr>
+</tbody>
+</table>
+</dd>
+<dt>dbexportcmd:</dt>
+<dd><table class="first last docutils field-list" frame="void" rules="none">
+<col class="field-name" />
+<col class="field-body" />
+<tbody valign="top">
+<tr class="field-odd field"><th class="field-name">mode:</th><td class="field-body">1 (simple), 2 (advance)</td>
+</tr>
+<tr class="field-even field"><th class="field-name">hive_table:</th><td class="field-body">Name of the hive table</td>
+</tr>
+<tr class="field-odd field"><th class="field-name">partition_spec:</th><td class="field-body">partition specification for Hive table.</td>
+</tr>
+<tr class="field-even field"><th class="field-name">dbtap_id:</th><td class="field-body">data store ID of the target database, in Qubole.</td>
+</tr>
+<tr class="field-odd field"><th class="field-name">db_table:</th><td class="field-body">name of the db table</td>
+</tr>
+<tr class="field-even field"><th class="field-name">db_update_mode:</th><td class="field-body">allowinsert or updateonly</td>
+</tr>
+<tr class="field-odd field"><th class="field-name">db_update_keys:</th><td class="field-body">columns used to determine the uniqueness of rows</td>
+</tr>
+<tr class="field-even field"><th class="field-name">export_dir:</th><td class="field-body">HDFS/S3 location from which data will be exported.</td>
+</tr>
+<tr class="field-odd field"><th class="field-name" colspan="2">fields_terminated_by:</th></tr>
+<tr class="field-odd field"><td>&nbsp;</td><td class="field-body">hex of the char used as column separator in the dataset.</td>
+</tr>
+</tbody>
+</table>
+</dd>
+<dt>dbimportcmd:</dt>
+<dd><table class="first last docutils field-list" frame="void" rules="none">
+<col class="field-name" />
+<col class="field-body" />
+<tbody valign="top">
+<tr class="field-odd field"><th class="field-name">mode:</th><td class="field-body">1 (simple), 2 (advance)</td>
+</tr>
+<tr class="field-even field"><th class="field-name">hive_table:</th><td class="field-body">Name of the hive table</td>
+</tr>
+<tr class="field-odd field"><th class="field-name">dbtap_id:</th><td class="field-body">data store ID of the target database, in Qubole.</td>
+</tr>
+<tr class="field-even field"><th class="field-name">db_table:</th><td class="field-body">name of the db table</td>
+</tr>
+<tr class="field-odd field"><th class="field-name">where_clause:</th><td class="field-body">where clause, if any</td>
+</tr>
+<tr class="field-even field"><th class="field-name">parallelism:</th><td class="field-body">number of parallel db connections to use for extracting data</td>
+</tr>
+<tr class="field-odd field"><th class="field-name">extract_query:</th><td class="field-body">SQL query to extract data from db. $CONDITIONS must be part of the where clause.</td>
+</tr>
+<tr class="field-even field"><th class="field-name">boundary_query:</th><td class="field-body">Query to be used get range of row IDs to be extracted</td>
+</tr>
+<tr class="field-odd field"><th class="field-name">split_column:</th><td class="field-body">Column used as row ID to split data into ranges (mode 2)</td>
+</tr>
+</tbody>
+</table>
+</dd>
+</dl>
+</dd>
+</dl>
+<div class="admonition note">
+<p class="first admonition-title">Note</p>
+<p class="last">Following fields are template-supported : <code class="docutils literal"><span class="pre">query</span></code>, <code class="docutils literal"><span class="pre">script_location</span></code>, <code class="docutils literal"><span class="pre">sub_command</span></code>, <code class="docutils literal"><span class="pre">script</span></code>, <code class="docutils literal"><span class="pre">files</span></code>,
+<code class="docutils literal"><span class="pre">archives</span></code>, <code class="docutils literal"><span class="pre">program</span></code>, <code class="docutils literal"><span class="pre">cmdline</span></code>, <code class="docutils literal"><span class="pre">sql</span></code>, <code class="docutils literal"><span class="pre">where_clause</span></code>, <code class="docutils literal"><span class="pre">extract_query</span></code>, <code class="docutils literal"><span class="pre">boundary_query</span></code>, <code class="docutils literal"><span class="pre">macros</span></code>, <code class="docutils literal"><span class="pre">tags</span></code>,
+<code class="docutils literal"><span class="pre">name</span></code>, <code class="docutils literal"><span class="pre">parameters</span></code>. You can also use <code class="docutils literal"><span class="pre">.txt</span></code> files for template driven use cases.</p>
+</div>
+</dd></dl>
+
+<dl class="class">
 <dt id="airflow.contrib.operators.hipchat_operator.HipChatAPIOperator">
 <em class="property">class </em><code class="descclassname">airflow.contrib.operators.hipchat_operator.</code><code class="descname">HipChatAPIOperator</code><span class="sig-paren">(</span><em>token</em>, <em>base_url='https://api.hipchat.com/v2'</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/contrib/operators/hipchat_operator.html#HipChatAPIOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.hipchat_operator.HipChatAPIOperator" title="Permalink to this definition">�</a></dt>
 <dd><p>Base HipChat Operator.
@@ -2223,12 +2453,12 @@ to be retried.</p>
 <col class="field-name" />
 <col class="field-body" />
 <tbody valign="top">
-<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>lock_for_update</strong> &#8211; if True, indicates that the database should</td>
+<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>lock_for_update</strong> &#8211; if True, indicates that the database should
+lock the TaskInstance (issuing a FOR UPDATE clause) until the
+session is committed.</td>
 </tr>
 </tbody>
 </table>
-<p>lock the TaskInstance (issuing a FOR UPDATE clause) until the session
-is committed.</p>
 </dd></dl>
 
 <dl class="method">
@@ -3145,9 +3375,108 @@ directory, files will be uploaded inside.</li>
 
 </dd></dl>
 
-<div class="section" id="community-contributed-hooks">
-<h3>Community contributed hooks<a class="headerlink" href="#community-contributed-hooks" title="Permalink to this headline">�</a></h3>
-<span class="target" id="module-airflow.contrib.hooks"></span><dl class="class">
+<div class="section" id="module-airflow.contrib.hooks">
+<span id="community-contributed-hooks"></span><h3>Community contributed hooks<a class="headerlink" href="#module-airflow.contrib.hooks" title="Permalink to this headline">�</a></h3>
+<dl class="class">
+<dt id="airflow.contrib.hooks.BigQueryHook">
+<em class="property">class </em><code class="descclassname">airflow.contrib.hooks.</code><code class="descname">BigQueryHook</code><span class="sig-paren">(</span><em>bigquery_conn_id='bigquery_default'</em>, <em>delegate_to=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/bigquery_hook.html#BigQueryHook"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.BigQueryHook" title="Permalink to this definition">�</a></dt>
+<dd><p>Bases: <code class="xref py py-class docutils literal"><span class="pre">airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook</span></code>, <code class="xref py py-class docutils literal"><span class="pre">airflow.hooks.dbapi_hook.DbApiHook</span></code></p>
+<p>Interact with BigQuery. This hook uses the Google Cloud Platform
+connection.</p>
+<dl class="method">
+<dt id="airflow.contrib.hooks.BigQueryHook.get_conn">
+<code class="descname">get_conn</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/bigquery_hook.html#BigQueryHook.get_conn"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.BigQueryHook.get_conn" title="Permalink to this definition">�</a></dt>
+<dd><p>Returns a BigQuery PEP 249 connection object.</p>
+</dd></dl>
+
+<dl class="method">
+<dt id="airflow.contrib.hooks.BigQueryHook.get_pandas_df">
+<code class="descname">get_pandas_df</code><span class="sig-paren">(</span><em>bql</em>, <em>parameters=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/bigquery_hook.html#BigQueryHook.get_pandas_df"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.BigQueryHook.get_pandas_df" title="Permalink to this definition">�</a></dt>
+<dd><p>Returns a Pandas DataFrame for the results produced by a BigQuery
+query. The DbApiHook method must be overridden because Pandas
+doesn&#8217;t support PEP 249 connections, except for SQLite. See:</p>
+<p><a class="reference external" href="https://github.com/pydata/pandas/blob/master/pandas/io/sql.py#L447">https://github.com/pydata/pandas/blob/master/pandas/io/sql.py#L447</a>
+<a class="reference external" href="https://github.com/pydata/pandas/issues/6900">https://github.com/pydata/pandas/issues/6900</a></p>
+<table class="docutils field-list" frame="void" rules="none">
+<col class="field-name" />
+<col class="field-body" />
+<tbody valign="top">
+<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>bql</strong> (<em>string</em>) &#8211; The BigQuery SQL to execute.</td>
+</tr>
+</tbody>
+</table>
+</dd></dl>
+
+<dl class="method">
+<dt id="airflow.contrib.hooks.BigQueryHook.get_service">
+<code class="descname">get_service</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/bigquery_hook.html#BigQueryHook.get_service"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.BigQueryHook.get_service" title="Permalink to this definition">�</a></dt>
+<dd><p>Returns a BigQuery service object.</p>
+</dd></dl>
+
+<dl class="method">
+<dt id="airflow.contrib.hooks.BigQueryHook.insert_rows">
+<code class="descname">insert_rows</code><span class="sig-paren">(</span><em>table</em>, <em>rows</em>, <em>target_fields=None</em>, <em>commit_every=1000</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/bigquery_hook.html#BigQueryHook.insert_rows"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.BigQueryHook.insert_rows" title="Permalink to this definition">�</a></dt>
+<dd><p>Insertion is currently unsupported. Theoretically, you could use
+BigQuery&#8217;s streaming API to insert rows into a table, but this hasn&#8217;t
+been implemented.</p>
+</dd></dl>
+
+</dd></dl>
+
+<dl class="class">
+<dt id="airflow.contrib.hooks.GoogleCloudStorageHook">
+<em class="property">class </em><code class="descclassname">airflow.contrib.hooks.</code><code class="descname">GoogleCloudStorageHook</code><span class="sig-paren">(</span><em>google_cloud_storage_conn_id='google_cloud_storage_default'</em>, <em>delegate_to=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/gcs_hook.html#GoogleCloudStorageHook"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.GoogleCloudStorageHook" title="Permalink to this definition">�</a></dt>
+<dd><p>Bases: <code class="xref py py-class docutils literal"><span class="pre">airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook</span></code></p>
+<p>Interact with Google Cloud Storage. This hook uses the Google Cloud Platform
+connection.</p>
+<dl class="method">
+<dt id="airflow.contrib.hooks.GoogleCloudStorageHook.download">
+<code class="descname">download</code><span class="sig-paren">(</span><em>bucket</em>, <em>object</em>, <em>filename=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/gcs_hook.html#GoogleCloudStorageHook.download"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.GoogleCloudStorageHook.download" title="Permalink to this definition">�</a></dt>
+<dd><p>Get a file from Google Cloud Storage.</p>
+<table class="docutils field-list" frame="void" rules="none">
+<col class="field-name" />
+<col class="field-body" />
+<tbody valign="top">
+<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple">
+<li><strong>bucket</strong> (<em>string</em>) &#8211; The bucket to fetch from.</li>
+<li><strong>object</strong> (<em>string</em>) &#8211; The object to fetch.</li>
+<li><strong>filename</strong> (<em>string</em>) &#8211; If set, a local file path where the file should be written to.</li>
+</ul>
+</td>
+</tr>
+</tbody>
+</table>
+</dd></dl>
+
+<dl class="method">
+<dt id="airflow.contrib.hooks.GoogleCloudStorageHook.get_conn">
+<code class="descname">get_conn</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/gcs_hook.html#GoogleCloudStorageHook.get_conn"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.GoogleCloudStorageHook.get_conn" title="Permalink to this definition">�</a></dt>
+<dd><p>Returns a Google Cloud Storage service object.</p>
+</dd></dl>
+
+<dl class="method">
+<dt id="airflow.contrib.hooks.GoogleCloudStorageHook.upload">
+<code class="descname">upload</code><span class="sig-paren">(</span><em>bucket</em>, <em>object</em>, <em>filename</em>, <em>mime_type='application/octet-stream'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/gcs_hook.html#GoogleCloudStorageHook.upload"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.GoogleCloudStorageHook.upload" title="Permalink to this definition">�</a></dt>
+<dd><p>Uploads a local file to Google Cloud Storage.</p>
+<table class="docutils field-list" frame="void" rules="none">
+<col class="field-name" />
+<col class="field-body" />
+<tbody valign="top">
+<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple">
+<li><strong>bucket</strong> (<em>string</em>) &#8211; The bucket to upload to.</li>
+<li><strong>object</strong> (<em>string</em>) &#8211; The object name to set when uploading the local file.</li>
+<li><strong>filename</strong> (<em>string</em>) &#8211; The local file path to the file to be uploaded.</li>
+<li><strong>mime_type</strong> (<em>string</em>) &#8211; The MIME type to set when uploading the file.</li>
+</ul>
+</td>
+</tr>
+</tbody>
+</table>
+</dd></dl>
+
+</dd></dl>
+
+<dl class="class">
 <dt id="airflow.contrib.hooks.VerticaHook">
 <em class="property">class </em><code class="descclassname">airflow.contrib.hooks.</code><code class="descname">VerticaHook</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/vertica_hook.html#VerticaHook"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.VerticaHook" title="Permalink to this definition">�</a></dt>
 <dd><p>Bases: <code class="xref py py-class docutils literal"><span class="pre">airflow.hooks.dbapi_hook.DbApiHook</span></code></p>
@@ -3406,6 +3735,13 @@ documentation <a class="reference external" href="https://github.com/cloudant-la
 
 </dd></dl>
 
+<dl class="class">
+<dt id="airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook">
+<em class="property">class </em><code class="descclassname">airflow.contrib.hooks.gcs_hook.</code><code class="descname">GoogleCloudStorageHook</code><span class="sig-paren">(</span><em>google_cloud_storage_conn_id='google_cloud_storage_default'</em>, <em>delegate_to=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/contrib/hooks/gcs_hook.html#GoogleCloudStorageHook"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook" title="Permalink to this definition">�</a></dt>
+<dd><p>Interact with Google Cloud Storage. This hook uses the Google Cloud Platform
+connection.</p>
+</dd></dl>
+
 </div>
 </div>
 <div class="section" id="executors">
@@ -3444,7 +3780,19 @@ SequentialExecutor alongside sqlite as you first install it.</p>
 
 <div class="section" id="community-contributed-executors">
 <h3>Community-contributed executors<a class="headerlink" href="#community-contributed-executors" title="Permalink to this headline">�</a></h3>
-<span class="target" id="module-airflow.contrib.executors"></span></div>
+<dl class="class">
+<dt id="airflow.contrib.executors.mesos_executor.MesosExecutor">
+<em class="property">class </em><code class="descclassname">airflow.contrib.executors.mesos_executor.</code><code class="descname">MesosExecutor</code><span class="sig-paren">(</span><em>parallelism=16</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/contrib/executors/mesos_executor.html#MesosExecutor"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.executors.mesos_executor.MesosExecutor" title="Permalink to this definition">�</a></dt>
+<dd><p>MesosExecutor allows distributing the execution of task
+instances to multiple mesos workers.</p>
+<p>Apache Mesos is a distributed systems kernel which abstracts
+CPU, memory, storage, and other compute resources away from
+machines (physical or virtual), enabling fault-tolerant and
+elastic distributed systems to easily be built and run effectively.
+See <a class="reference external" href="http://mesos.apache.org/">http://mesos.apache.org/</a></p>
+</dd></dl>
+
+</div>
 </div>
 </div>
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/502934ca/concepts.html
----------------------------------------------------------------------
diff --git a/concepts.html b/concepts.html
index 5fda645..b41766f 100644
--- a/concepts.html
+++ b/concepts.html
@@ -748,6 +748,20 @@ content if defined:</p>
 <p>This is especially useful if your tasks are built dynamically from
 configuration files, it allows you to expose the configuration that led
 to the related tasks in Airflow.</p>
+<div class="code python highlight-default"><div class="highlight"><pre><span></span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">### My great DAG</span>
+<span class="sd">&quot;&quot;&quot;</span>
+
+<span class="n">dag</span> <span class="o">=</span> <span class="n">DAG</span><span class="p">(</span><span class="s1">&#39;my_dag&#39;</span><span class="p">,</span> <span class="n">default_args</span><span class="o">=</span><span class="n">default_args</span><span class="p">)</span>
+<span class="n">dag</span><span class="o">.</span><span class="n">doc_md</span> <span class="o">=</span> <span class="n">__doc__</span>
+
+<span class="n">t</span> <span class="o">=</span> <span class="n">BashOperator</span><span class="p">(</span><span class="s2">&quot;foo&quot;</span><span class="p">,</span> <span class="n">dag</span><span class="o">=</span><span class="n">dag</span><span class="p">)</span>
+<span class="n">t</span><span class="o">.</span><span class="n">doc_md</span> <span class="o">=</span> <span class="s2">&quot;&quot;&quot;</span><span class="se">\</span>
+<span class="s2">#Title&quot;</span>
+<span class="s2">Here&#39;s a [url](www.airbnb.com)</span>
+<span class="s2">&quot;&quot;&quot;</span>
+</pre></div>
+</div>
 <p>This content will get rendered as markdown respectively in the &#8220;Graph View&#8221; and
 &#8220;Task Details&#8221; pages.</p>
 </div>

[6/6] incubator-airflow-site git commit: Adding incubator related disclaimer

[ma...@apache.org]

Adding incubator related disclaimer


Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/commit/502934ca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/tree/502934ca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/diff/502934ca

Branch: refs/heads/asf-site
Commit: 502934ca871cea1ba853467bcdc505948bc73173
Parents: 1bb5481
Author: Maxime Beauchemin <ma...@apache.org>
Authored: Wed Jun 29 09:14:36 2016 -0700
Committer: Maxime Beauchemin <ma...@apache.org>
Committed: Wed Jun 29 09:15:01 2016 -0700

----------------------------------------------------------------------
 .../contrib/executors/mesos_executor.html       | 470 +++++++++
 _modules/airflow/contrib/hooks/gcs_hook.html    | 284 ++++++
 .../contrib/operators/bigquery_operator.html    | 258 +++++
 .../contrib/operators/bigquery_to_gcs.html      | 278 ++++++
 .../operators/gcs_download_operator.html        | 253 +++++
 _modules/airflow/models.html                    |  20 +-
 _modules/bigquery_hook.html                     | 995 +++++++++++++++++++
 _modules/gcs_hook.html                          |  17 +-
 _modules/index.html                             |   7 +
 _modules/qubole_operator.html                   | 328 ++++++
 _sources/code.txt                               |  13 +-
 _sources/concepts.txt                           |   1 +
 _sources/index.txt                              |  15 +-
 cli.html                                        |  83 +-
 code.html                                       | 372 ++++++-
 concepts.html                                   |  14 +
 genindex.html                                   |  91 +-
 index.html                                      |  17 +-
 objects.inv                                     | Bin 2103 -> 2236 bytes
 py-modindex.html                                |   5 -
 searchindex.js                                  |   2 +-
 21 files changed, 3446 insertions(+), 77 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/502934ca/_modules/airflow/contrib/executors/mesos_executor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/executors/mesos_executor.html b/_modules/airflow/contrib/executors/mesos_executor.html
new file mode 100644
index 0000000..65557d9
--- /dev/null
+++ b/_modules/airflow/contrib/executors/mesos_executor.html
@@ -0,0 +1,470 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>airflow.contrib.executors.mesos_executor &mdash; Airflow Documentation</title>
+  
+
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+    <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+        <link rel="up" title="Module code" href="../../../index.html"/> 
+
+  
+  <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+                <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+        <a href="../../../../index.html">Airflow</a>
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+  <ul class="wy-breadcrumbs">
+    <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+      
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+      
+    <li>airflow.contrib.executors.mesos_executor</li>
+      <li class="wy-breadcrumbs-aside">
+        
+          
+        
+      </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for airflow.contrib.executors.mesos_executor</h1><div class="highlight"><pre>
+<span></span><span class="kn">from</span> <span class="nn">future</span> <span class="kn">import</span> <span class="n">standard_library</span>
+<span class="n">standard_library</span><span class="o">.</span><span class="n">install_aliases</span><span class="p">()</span>
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
+<span class="kn">import</span> <span class="nn">logging</span>
+<span class="kn">from</span> <span class="nn">queue</span> <span class="kn">import</span> <span class="n">Queue</span>
+
+<span class="kn">import</span> <span class="nn">mesos.interface</span>
+<span class="kn">from</span> <span class="nn">mesos.interface</span> <span class="kn">import</span> <span class="n">mesos_pb2</span>
+<span class="kn">import</span> <span class="nn">mesos.native</span>
+
+<span class="kn">from</span> <span class="nn">airflow</span> <span class="kn">import</span> <span class="n">configuration</span>
+<span class="kn">from</span> <span class="nn">airflow.executors.base_executor</span> <span class="kn">import</span> <span class="n">BaseExecutor</span>
+<span class="kn">from</span> <span class="nn">airflow.settings</span> <span class="kn">import</span> <span class="n">Session</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="kn">import</span> <span class="n">State</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span>
+
+
+<span class="n">DEFAULT_FRAMEWORK_NAME</span> <span class="o">=</span> <span class="s1">&#39;Airflow&#39;</span>
+<span class="n">FRAMEWORK_CONNID_PREFIX</span> <span class="o">=</span> <span class="s1">&#39;mesos_framework_&#39;</span>
+
+
+<span class="k">def</span> <span class="nf">get_framework_name</span><span class="p">():</span>
+    <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;FRAMEWORK_NAME&#39;</span><span class="p">):</span>
+        <span class="k">return</span> <span class="n">DEFAULT_FRAMEWORK_NAME</span>
+    <span class="k">return</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;FRAMEWORK_NAME&#39;</span><span class="p">)</span>
+
+
+<span class="c1"># AirflowMesosScheduler, implements Mesos Scheduler interface</span>
+<span class="c1"># To schedule airflow jobs on mesos</span>
+<span class="k">class</span> <span class="nc">AirflowMesosScheduler</span><span class="p">(</span><span class="n">mesos</span><span class="o">.</span><span class="n">interface</span><span class="o">.</span><span class="n">Scheduler</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Airflow Mesos scheduler implements mesos scheduler interface</span>
+<span class="sd">    to schedule airflow tasks on mesos.</span>
+<span class="sd">    Basically, it schedules a command like</span>
+<span class="sd">    &#39;airflow run &lt;dag_id&gt; &lt;task_instance_id&gt; &lt;start_date&gt; --local -p=&lt;pickle&gt;&#39;</span>
+<span class="sd">    to run on a mesos slave.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+                 <span class="n">task_queue</span><span class="p">,</span>
+                 <span class="n">result_queue</span><span class="p">,</span>
+                 <span class="n">task_cpu</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
+                 <span class="n">task_mem</span><span class="o">=</span><span class="mi">256</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span> <span class="o">=</span> <span class="n">task_queue</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span> <span class="o">=</span> <span class="n">result_queue</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">task_cpu</span> <span class="o">=</span> <span class="n">task_cpu</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">task_mem</span> <span class="o">=</span> <span class="n">task_mem</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">task_counter</span> <span class="o">=</span> <span class="mi">0</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">task_key_map</span> <span class="o">=</span> <span class="p">{}</span>
+
+    <span class="k">def</span> <span class="nf">registered</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">frameworkId</span><span class="p">,</span> <span class="n">masterInfo</span><span class="p">):</span>
+        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;AirflowScheduler registered to mesos with framework ID </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">frameworkId</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;CHECKPOINT&#39;</span><span class="p">)</span> <span class="ow">and</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;FAILOVER_TIMEOUT&#39;</span><span class="p">):</span>
+            <span class="c1"># Import here to work around a circular import error</span>
+            <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">Connection</span>
+
+            <span class="c1"># Update the Framework ID in the database.</span>
+            <span class="n">session</span> <span class="o">=</span> <span class="n">Session</span><span class="p">()</span>
+            <span class="n">conn_id</span> <span class="o">=</span> <span class="n">FRAMEWORK_CONNID_PREFIX</span> <span class="o">+</span> <span class="n">get_framework_name</span><span class="p">()</span>
+            <span class="n">connection</span> <span class="o">=</span> <span class="n">Session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">Connection</span><span class="p">)</span><span class="o">.</span><span class="n">filter_by</span><span class="p">(</span><span class="n">conn_id</span><span class="o">=</span><span class="n">conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">first</span><span class="p">()</span>
+            <span class="k">if</span> <span class="n">connection</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
+                <span class="n">connection</span> <span class="o">=</span> <span class="n">Connection</span><span class="p">(</span><span class="n">conn_id</span><span class="o">=</span><span class="n">conn_id</span><span class="p">,</span> <span class="n">conn_type</span><span class="o">=</span><span class="s1">&#39;mesos_framework-id&#39;</span><span class="p">,</span>
+                                        <span class="n">extra</span><span class="o">=</span><span class="n">frameworkId</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">connection</span><span class="o">.</span><span class="n">extra</span> <span class="o">=</span> <span class="n">frameworkId</span><span class="o">.</span><span class="n">value</span>
+
+            <span class="n">session</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">connection</span><span class="p">)</span>
+            <span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
+            <span class="n">Session</span><span class="o">.</span><span class="n">remove</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">reregistered</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">masterInfo</span><span class="p">):</span>
+        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;AirflowScheduler re-registered to mesos&quot;</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">disconnected</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">):</span>
+        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;AirflowScheduler disconnected from mesos&quot;</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">offerRescinded</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">offerId</span><span class="p">):</span>
+        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;AirflowScheduler offer </span><span class="si">%s</span><span class="s2"> rescinded&quot;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">offerId</span><span class="p">))</span>
+
+    <span class="k">def</span> <span class="nf">frameworkMessage</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">executorId</span><span class="p">,</span> <span class="n">slaveId</span><span class="p">,</span> <span class="n">message</span><span class="p">):</span>
+        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;AirflowScheduler received framework message </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">message</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">executorLost</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">executorId</span><span class="p">,</span> <span class="n">slaveId</span><span class="p">,</span> <span class="n">status</span><span class="p">):</span>
+        <span class="n">logging</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">&quot;AirflowScheduler executor </span><span class="si">%s</span><span class="s2"> lost&quot;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">executorId</span><span class="p">))</span>
+
+    <span class="k">def</span> <span class="nf">slaveLost</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">slaveId</span><span class="p">):</span>
+        <span class="n">logging</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">&quot;AirflowScheduler slave </span><span class="si">%s</span><span class="s2"> lost&quot;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">slaveId</span><span class="p">))</span>
+
+    <span class="k">def</span> <span class="nf">error</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">message</span><span class="p">):</span>
+        <span class="n">logging</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;AirflowScheduler driver aborted </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">message</span><span class="p">)</span>
+        <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">&quot;AirflowScheduler driver aborted </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">message</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">resourceOffers</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">offers</span><span class="p">):</span>
+        <span class="k">for</span> <span class="n">offer</span> <span class="ow">in</span> <span class="n">offers</span><span class="p">:</span>
+            <span class="n">tasks</span> <span class="o">=</span> <span class="p">[]</span>
+            <span class="n">offerCpus</span> <span class="o">=</span> <span class="mi">0</span>
+            <span class="n">offerMem</span> <span class="o">=</span> <span class="mi">0</span>
+            <span class="k">for</span> <span class="n">resource</span> <span class="ow">in</span> <span class="n">offer</span><span class="o">.</span><span class="n">resources</span><span class="p">:</span>
+                <span class="k">if</span> <span class="n">resource</span><span class="o">.</span><span class="n">name</span> <span class="o">==</span> <span class="s2">&quot;cpus&quot;</span><span class="p">:</span>
+                    <span class="n">offerCpus</span> <span class="o">+=</span> <span class="n">resource</span><span class="o">.</span><span class="n">scalar</span><span class="o">.</span><span class="n">value</span>
+                <span class="k">elif</span> <span class="n">resource</span><span class="o">.</span><span class="n">name</span> <span class="o">==</span> <span class="s2">&quot;mem&quot;</span><span class="p">:</span>
+                    <span class="n">offerMem</span> <span class="o">+=</span> <span class="n">resource</span><span class="o">.</span><span class="n">scalar</span><span class="o">.</span><span class="n">value</span>
+
+            <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Received offer </span><span class="si">%s</span><span class="s2"> with cpus: </span><span class="si">%s</span><span class="s2"> and mem: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">offer</span><span class="o">.</span><span class="n">id</span><span class="o">.</span><span class="n">value</span><span class="p">,</span> <span class="n">offerCpus</span><span class="p">,</span> <span class="n">offerMem</span><span class="p">)</span>
+
+            <span class="n">remainingCpus</span> <span class="o">=</span> <span class="n">offerCpus</span>
+            <span class="n">remainingMem</span> <span class="o">=</span> <span class="n">offerMem</span>
+
+            <span class="k">while</span> <span class="p">(</span><span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">empty</span><span class="p">())</span> <span class="ow">and</span> \
+                  <span class="n">remainingCpus</span> <span class="o">&gt;=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_cpu</span> <span class="ow">and</span> \
+                  <span class="n">remainingMem</span> <span class="o">&gt;=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_mem</span><span class="p">:</span>
+                <span class="n">key</span><span class="p">,</span> <span class="n">cmd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
+                <span class="n">tid</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_counter</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">task_counter</span> <span class="o">+=</span> <span class="mi">1</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">task_key_map</span><span class="p">[</span><span class="nb">str</span><span class="p">(</span><span class="n">tid</span><span class="p">)]</span> <span class="o">=</span> <span class="n">key</span>
+
+                <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Launching task </span><span class="si">%d</span><span class="s2"> using offer </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">tid</span><span class="p">,</span> <span class="n">offer</span><span class="o">.</span><span class="n">id</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
+
+                <span class="n">task</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TaskInfo</span><span class="p">()</span>
+                <span class="n">task</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">tid</span><span class="p">)</span>
+                <span class="n">task</span><span class="o">.</span><span class="n">slave_id</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="n">offer</span><span class="o">.</span><span class="n">slave_id</span><span class="o">.</span><span class="n">value</span>
+                <span class="n">task</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="s2">&quot;AirflowTask </span><span class="si">%d</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">tid</span>
+
+                <span class="n">cpus</span> <span class="o">=</span> <span class="n">task</span><span class="o">.</span><span class="n">resources</span><span class="o">.</span><span class="n">add</span><span class="p">()</span>
+                <span class="n">cpus</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="s2">&quot;cpus&quot;</span>
+                <span class="n">cpus</span><span class="o">.</span><span class="n">type</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">Value</span><span class="o">.</span><span class="n">SCALAR</span>
+                <span class="n">cpus</span><span class="o">.</span><span class="n">scalar</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_cpu</span>
+
+                <span class="n">mem</span> <span class="o">=</span> <span class="n">task</span><span class="o">.</span><span class="n">resources</span><span class="o">.</span><span class="n">add</span><span class="p">()</span>
+                <span class="n">mem</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="s2">&quot;mem&quot;</span>
+                <span class="n">mem</span><span class="o">.</span><span class="n">type</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">Value</span><span class="o">.</span><span class="n">SCALAR</span>
+                <span class="n">mem</span><span class="o">.</span><span class="n">scalar</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_mem</span>
+
+                <span class="n">command</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">CommandInfo</span><span class="p">()</span>
+                <span class="n">command</span><span class="o">.</span><span class="n">shell</span> <span class="o">=</span> <span class="bp">True</span>
+                <span class="n">command</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="n">cmd</span>
+                <span class="n">task</span><span class="o">.</span><span class="n">command</span><span class="o">.</span><span class="n">MergeFrom</span><span class="p">(</span><span class="n">command</span><span class="p">)</span>
+
+                <span class="n">tasks</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">task</span><span class="p">)</span>
+
+                <span class="n">remainingCpus</span> <span class="o">-=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_cpu</span>
+                <span class="n">remainingMem</span> <span class="o">-=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_mem</span>
+
+            <span class="n">driver</span><span class="o">.</span><span class="n">launchTasks</span><span class="p">(</span><span class="n">offer</span><span class="o">.</span><span class="n">id</span><span class="p">,</span> <span class="n">tasks</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">statusUpdate</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">driver</span><span class="p">,</span> <span class="n">update</span><span class="p">):</span>
+        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Task </span><span class="si">%s</span><span class="s2"> is in state </span><span class="si">%s</span><span class="s2">, data </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span>
+                     <span class="n">update</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">value</span><span class="p">,</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TaskState</span><span class="o">.</span><span class="n">Name</span><span class="p">(</span><span class="n">update</span><span class="o">.</span><span class="n">state</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">update</span><span class="o">.</span><span class="n">data</span><span class="p">))</span>
+
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">key</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_key_map</span><span class="p">[</span><span class="n">update</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">value</span><span class="p">]</span>
+        <span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span>
+            <span class="c1"># The map may not contain an item if the framework re-registered after a failover.</span>
+            <span class="c1"># Discard these tasks.</span>
+            <span class="n">logging</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s2">&quot;Unrecognised task key </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">update</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
+            <span class="k">return</span>
+
+        <span class="k">if</span> <span class="n">update</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TASK_FINISHED</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">State</span><span class="o">.</span><span class="n">SUCCESS</span><span class="p">))</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">task_done</span><span class="p">()</span>
+
+        <span class="k">if</span> <span class="n">update</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TASK_LOST</span> <span class="ow">or</span> \
+           <span class="n">update</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TASK_KILLED</span> <span class="ow">or</span> \
+           <span class="n">update</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TASK_FAILED</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">State</span><span class="o">.</span><span class="n">FAILED</span><span class="p">))</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">task_done</span><span class="p">()</span>
+
+
+<div class="viewcode-block" id="MesosExecutor"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.executors.mesos_executor.MesosExecutor">[docs]</a><span class="k">class</span> <span class="nc">MesosExecutor</span><span class="p">(</span><span class="n">BaseExecutor</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    MesosExecutor allows distributing the execution of task</span>
+<span class="sd">    instances to multiple mesos workers.</span>
+
+<span class="sd">    Apache Mesos is a distributed systems kernel which abstracts</span>
+<span class="sd">    CPU, memory, storage, and other compute resources away from</span>
+<span class="sd">    machines (physical or virtual), enabling fault-tolerant and</span>
+<span class="sd">    elastic distributed systems to easily be built and run effectively.</span>
+<span class="sd">    See http://mesos.apache.org/</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="nf">start</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span> <span class="o">=</span> <span class="n">Queue</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span> <span class="o">=</span> <span class="n">Queue</span><span class="p">()</span>
+        <span class="n">framework</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">FrameworkInfo</span><span class="p">()</span>
+        <span class="n">framework</span><span class="o">.</span><span class="n">user</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span>
+
+        <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;MASTER&#39;</span><span class="p">):</span>
+            <span class="n">logging</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;Expecting mesos master URL for mesos executor&quot;</span><span class="p">)</span>
+            <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">&quot;mesos.master not provided for mesos executor&quot;</span><span class="p">)</span>
+
+        <span class="n">master</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;MASTER&#39;</span><span class="p">)</span>
+
+        <span class="n">framework</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="n">get_framework_name</span><span class="p">()</span>
+
+        <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;TASK_CPU&#39;</span><span class="p">):</span>
+            <span class="n">task_cpu</span> <span class="o">=</span> <span class="mi">1</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">task_cpu</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;TASK_CPU&#39;</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;TASK_MEMORY&#39;</span><span class="p">):</span>
+            <span class="n">task_memory</span> <span class="o">=</span> <span class="mi">256</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">task_memory</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;TASK_MEMORY&#39;</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;CHECKPOINT&#39;</span><span class="p">):</span>
+            <span class="n">framework</span><span class="o">.</span><span class="n">checkpoint</span> <span class="o">=</span> <span class="bp">True</span>
+
+            <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;FAILOVER_TIMEOUT&#39;</span><span class="p">):</span>
+                <span class="c1"># Import here to work around a circular import error</span>
+                <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">Connection</span>
+
+                <span class="c1"># Query the database to get the ID of the Mesos Framework, if available.</span>
+                <span class="n">conn_id</span> <span class="o">=</span> <span class="n">FRAMEWORK_CONNID_PREFIX</span> <span class="o">+</span> <span class="n">framework</span><span class="o">.</span><span class="n">name</span>
+                <span class="n">session</span> <span class="o">=</span> <span class="n">Session</span><span class="p">()</span>
+                <span class="n">connection</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">Connection</span><span class="p">)</span><span class="o">.</span><span class="n">filter_by</span><span class="p">(</span><span class="n">conn_id</span><span class="o">=</span><span class="n">conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">first</span><span class="p">()</span>
+                <span class="k">if</span> <span class="n">connection</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span>
+                    <span class="c1"># Set the Framework ID to let the scheduler reconnect with running tasks.</span>
+                    <span class="n">framework</span><span class="o">.</span><span class="n">id</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="n">connection</span><span class="o">.</span><span class="n">extra</span>
+
+                <span class="n">framework</span><span class="o">.</span><span class="n">failover_timeout</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;FAILOVER_TIMEOUT&#39;</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">framework</span><span class="o">.</span><span class="n">checkpoint</span> <span class="o">=</span> <span class="bp">False</span>
+
+        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;MesosFramework master : </span><span class="si">%s</span><span class="s1">, name : </span><span class="si">%s</span><span class="s1">, cpu : </span><span class="si">%s</span><span class="s1">, mem : </span><span class="si">%s</span><span class="s1">, checkpoint : </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span>
+            <span class="n">master</span><span class="p">,</span> <span class="n">framework</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">task_cpu</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">task_memory</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">framework</span><span class="o">.</span><span class="n">checkpoint</span><span class="p">))</span>
+
+        <span class="n">implicit_acknowledgements</span> <span class="o">=</span> <span class="mi">1</span>
+
+        <span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;AUTHENTICATE&#39;</span><span class="p">):</span>
+            <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;DEFAULT_PRINCIPAL&#39;</span><span class="p">):</span>
+                <span class="n">logging</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;Expecting authentication principal in the environment&quot;</span><span class="p">)</span>
+                <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">&quot;mesos.default_principal not provided in authenticated mode&quot;</span><span class="p">)</span>
+            <span class="k">if</span> <span class="ow">not</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;DEFAULT_SECRET&#39;</span><span class="p">):</span>
+                <span class="n">logging</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;Expecting authentication secret in the environment&quot;</span><span class="p">)</span>
+                <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">&quot;mesos.default_secret not provided in authenticated mode&quot;</span><span class="p">)</span>
+
+            <span class="n">credential</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">Credential</span><span class="p">()</span>
+            <span class="n">credential</span><span class="o">.</span><span class="n">principal</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;DEFAULT_PRINCIPAL&#39;</span><span class="p">)</span>
+            <span class="n">credential</span><span class="o">.</span><span class="n">secret</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;mesos&#39;</span><span class="p">,</span> <span class="s1">&#39;DEFAULT_SECRET&#39;</span><span class="p">)</span>
+
+            <span class="n">framework</span><span class="o">.</span><span class="n">principal</span> <span class="o">=</span> <span class="n">credential</span><span class="o">.</span><span class="n">principal</span>
+
+            <span class="n">driver</span> <span class="o">=</span> <span class="n">mesos</span><span class="o">.</span><span class="n">native</span><span class="o">.</span><span class="n">MesosSchedulerDriver</span><span class="p">(</span>
+                <span class="n">AirflowMesosScheduler</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="p">,</span> <span class="n">task_cpu</span><span class="p">,</span> <span class="n">task_memory</span><span class="p">),</span>
+                <span class="n">framework</span><span class="p">,</span>
+                <span class="n">master</span><span class="p">,</span>
+                <span class="n">implicit_acknowledgements</span><span class="p">,</span>
+                <span class="n">credential</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">framework</span><span class="o">.</span><span class="n">principal</span> <span class="o">=</span> <span class="s1">&#39;Airflow&#39;</span>
+            <span class="n">driver</span> <span class="o">=</span> <span class="n">mesos</span><span class="o">.</span><span class="n">native</span><span class="o">.</span><span class="n">MesosSchedulerDriver</span><span class="p">(</span>
+                <span class="n">AirflowMesosScheduler</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="p">,</span> <span class="n">task_cpu</span><span class="p">,</span> <span class="n">task_memory</span><span class="p">),</span>
+                <span class="n">framework</span><span class="p">,</span>
+                <span class="n">master</span><span class="p">,</span>
+                <span class="n">implicit_acknowledgements</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">mesos_driver</span> <span class="o">=</span> <span class="n">driver</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mesos_driver</span><span class="o">.</span><span class="n">start</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">execute_async</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">queue</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">))</span>
+
+    <span class="k">def</span> <span class="nf">sync</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">while</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">empty</span><span class="p">():</span>
+            <span class="n">results</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">change_state</span><span class="p">(</span><span class="o">*</span><span class="n">results</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">end</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">join</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mesos_driver</span><span class="o">.</span><span class="n">stop</span><span class="p">()</span></div>
+</pre></div>
+
+           </div>
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'../../../../',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true
+        };
+    </script>
+      <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+      <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+      <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/502934ca/_modules/airflow/contrib/hooks/gcs_hook.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/hooks/gcs_hook.html b/_modules/airflow/contrib/hooks/gcs_hook.html
new file mode 100644
index 0000000..8edc584
--- /dev/null
+++ b/_modules/airflow/contrib/hooks/gcs_hook.html
@@ -0,0 +1,284 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>airflow.contrib.hooks.gcs_hook &mdash; Airflow Documentation</title>
+  
+
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+    <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+        <link rel="up" title="Module code" href="../../../index.html"/> 
+
+  
+  <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+                <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+        <a href="../../../../index.html">Airflow</a>
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+  <ul class="wy-breadcrumbs">
+    <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+      
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+      
+    <li>airflow.contrib.hooks.gcs_hook</li>
+      <li class="wy-breadcrumbs-aside">
+        
+          
+        
+      </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for airflow.contrib.hooks.gcs_hook</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+<span class="c1">#</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcp_api_base_hook</span> <span class="kn">import</span> <span class="n">GoogleCloudBaseHook</span>
+<span class="kn">from</span> <span class="nn">apiclient.discovery</span> <span class="kn">import</span> <span class="n">build</span>
+<span class="kn">from</span> <span class="nn">apiclient.http</span> <span class="kn">import</span> <span class="n">MediaFileUpload</span>
+
+<span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">&quot;google_cloud_storage&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">INFO</span><span class="p">)</span>
+
+
+<div class="viewcode-block" id="GoogleCloudStorageHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook">[docs]</a><span class="k">class</span> <span class="nc">GoogleCloudStorageHook</span><span class="p">(</span><span class="n">GoogleCloudBaseHook</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Interact with Google Cloud Storage. This hook uses the Google Cloud Platform</span>
+<span class="sd">    connection.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+                 <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="s1">&#39;google_cloud_storage_default&#39;</span><span class="p">,</span>
+                 <span class="n">delegate_to</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">GoogleCloudStorageHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="n">google_cloud_storage_conn_id</span><span class="p">,</span> <span class="n">delegate_to</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Returns a Google Cloud Storage service object.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">http_authorized</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_authorize</span><span class="p">()</span>
+        <span class="k">return</span> <span class="n">build</span><span class="p">(</span><span class="s1">&#39;storage&#39;</span><span class="p">,</span> <span class="s1">&#39;v1&#39;</span><span class="p">,</span> <span class="n">http</span><span class="o">=</span><span class="n">http_authorized</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">download</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="nb">object</span><span class="p">,</span> <span class="n">filename</span><span class="o">=</span><span class="bp">False</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Get a file from Google Cloud Storage.</span>
+
+<span class="sd">        :param bucket: The bucket to fetch from.</span>
+<span class="sd">        :type bucket: string</span>
+<span class="sd">        :param object: The object to fetch.</span>
+<span class="sd">        :type object: string</span>
+<span class="sd">        :param filename: If set, a local file path where the file should be written to.</span>
+<span class="sd">        :type filename: string</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+        <span class="n">downloaded_file_bytes</span> <span class="o">=</span> <span class="n">service</span> \
+            <span class="o">.</span><span class="n">objects</span><span class="p">()</span> \
+            <span class="o">.</span><span class="n">get_media</span><span class="p">(</span><span class="n">bucket</span><span class="o">=</span><span class="n">bucket</span><span class="p">,</span> <span class="nb">object</span><span class="o">=</span><span class="nb">object</span><span class="p">)</span> \
+            <span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+
+        <span class="c1"># Write the file to local file path, if requested.</span>
+        <span class="k">if</span> <span class="n">filename</span><span class="p">:</span>
+            <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="s1">&#39;w&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">file_fd</span><span class="p">:</span>
+                <span class="n">file_fd</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">downloaded_file_bytes</span><span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">downloaded_file_bytes</span>
+
+    <span class="k">def</span> <span class="nf">upload</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="nb">object</span><span class="p">,</span> <span class="n">filename</span><span class="p">,</span> <span class="n">mime_type</span><span class="o">=</span><span class="s1">&#39;application/octet-stream&#39;</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Uploads a local file to Google Cloud Storage.</span>
+
+<span class="sd">        :param bucket: The bucket to upload to.</span>
+<span class="sd">        :type bucket: string</span>
+<span class="sd">        :param object: The object name to set when uploading the local file.</span>
+<span class="sd">        :type object: string</span>
+<span class="sd">        :param filename: The local file path to the file to be uploaded.</span>
+<span class="sd">        :type filename: string</span>
+<span class="sd">        :param mime_type: The MIME type to set when uploading the file.</span>
+<span class="sd">        :type mime_type: string</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">service</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+        <span class="n">media</span> <span class="o">=</span> <span class="n">MediaFileUpload</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="n">mime_type</span><span class="p">)</span>
+        <span class="n">response</span> <span class="o">=</span> <span class="n">service</span> \
+            <span class="o">.</span><span class="n">objects</span><span class="p">()</span> \
+            <span class="o">.</span><span class="n">insert</span><span class="p">(</span><span class="n">bucket</span><span class="o">=</span><span class="n">bucket</span><span class="p">,</span> <span class="n">name</span><span class="o">=</span><span class="nb">object</span><span class="p">,</span> <span class="n">media_body</span><span class="o">=</span><span class="n">media</span><span class="p">)</span> \
+            <span class="o">.</span><span class="n">execute</span><span class="p">()</span></div>
+</pre></div>
+
+           </div>
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'../../../../',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true
+        };
+    </script>
+      <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+      <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+      <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/502934ca/_modules/airflow/contrib/operators/bigquery_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/bigquery_operator.html b/_modules/airflow/contrib/operators/bigquery_operator.html
new file mode 100644
index 0000000..830fd57
--- /dev/null
+++ b/_modules/airflow/contrib/operators/bigquery_operator.html
@@ -0,0 +1,258 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>airflow.contrib.operators.bigquery_operator &mdash; Airflow Documentation</title>
+  
+
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+    <link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
+        <link rel="up" title="Module code" href="../../../index.html"/> 
+
+  
+  <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../index.html" class="icon icon-home"> Airflow
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+                <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../configuration.html">Configuration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+        <a href="../../../../index.html">Airflow</a>
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+  <ul class="wy-breadcrumbs">
+    <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+      
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+      
+    <li>airflow.contrib.operators.bigquery_operator</li>
+      <li class="wy-breadcrumbs-aside">
+        
+          
+        
+      </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for airflow.contrib.operators.bigquery_operator</h1><div class="highlight"><pre>
+<span></span><span class="kn">import</span> <span class="nn">logging</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.bigquery_hook</span> <span class="kn">import</span> <span class="n">BigQueryHook</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
+
+<div class="viewcode-block" id="BigQueryOperator"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.bigquery_operator.BigQueryOperator">[docs]</a><span class="k">class</span> <span class="nc">BigQueryOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Executes BigQuery SQL queries in a specific BigQuery database</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;bql&#39;</span><span class="p">,</span> <span class="s1">&#39;destination_dataset_table&#39;</span><span class="p">)</span>
+    <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;.sql&#39;</span><span class="p">,)</span>
+    <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#e4f0e8&#39;</span>
+
+    <span class="nd">@apply_defaults</span>
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+                 <span class="n">bql</span><span class="p">,</span>
+                 <span class="n">destination_dataset_table</span> <span class="o">=</span> <span class="bp">False</span><span class="p">,</span>
+                 <span class="n">write_disposition</span> <span class="o">=</span> <span class="s1">&#39;WRITE_EMPTY&#39;</span><span class="p">,</span>
+                 <span class="n">allow_large_results</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
+                 <span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">&#39;bigquery_default&#39;</span><span class="p">,</span>
+                 <span class="n">delegate_to</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+                 <span class="n">udf_config</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
+                 <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+                 <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Create a new BigQueryOperator.</span>
+
+<span class="sd">        :param bql: the sql code to be executed</span>
+<span class="sd">        :type bql: Can receive a str representing a sql statement,</span>
+<span class="sd">            a list of str (sql statements), or reference to a template file.</span>
+<span class="sd">            Template reference are recognized by str ending in &#39;.sql&#39;</span>
+<span class="sd">        :param destination_dataset_table: A dotted dataset.table that, if set,</span>
+<span class="sd">            will store the results of the query.</span>
+<span class="sd">        :type destination_dataset_table: string</span>
+<span class="sd">        :param bigquery_conn_id: reference to a specific BigQuery hook.</span>
+<span class="sd">        :type bigquery_conn_id: string</span>
+<span class="sd">        :param delegate_to: The account to impersonate, if any.</span>
+<span class="sd">            For this to work, the service account making the request must have domain-wide delegation enabled.</span>
+<span class="sd">        :type delegate_to: string</span>
+<span class="sd">        :param udf_config: The User Defined Function configuration for the query.</span>
+<span class="sd">            See https://cloud.google.com/bigquery/user-defined-functions for details.</span>
+<span class="sd">        :type udf_config: list</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">BigQueryOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">bql</span> <span class="o">=</span> <span class="n">bql</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">destination_dataset_table</span> <span class="o">=</span> <span class="n">destination_dataset_table</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">write_disposition</span> <span class="o">=</span> <span class="n">write_disposition</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">allow_large_results</span> <span class="o">=</span> <span class="n">allow_large_results</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span> <span class="o">=</span> <span class="n">bigquery_conn_id</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">udf_config</span> <span class="o">=</span> <span class="n">udf_config</span>
+
+    <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Executing: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bql</span><span class="p">))</span>
+        <span class="n">hook</span> <span class="o">=</span> <span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span><span class="p">,</span> <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
+        <span class="n">conn</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+        <span class="n">cursor</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span>
+        <span class="n">cursor</span><span class="o">.</span><span class="n">run_query</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bql</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">destination_dataset_table</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">write_disposition</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">allow_large_results</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">udf_config</span><span class="p">)</span></div>
+</pre></div>
+
+           </div>
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'../../../../',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true
+        };
+    </script>
+      <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+      <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+      <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file