You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ma...@apache.org on 2017/03/06 17:10:22 UTC
[17/22] incubator-airflow-site git commit: Latest docs version as of
1.8.x
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/9c75ee9e/_modules/airflow/models.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/models.html b/_modules/airflow/models.html
index 2f044a4..a94686b 100644
--- a/_modules/airflow/models.html
+++ b/_modules/airflow/models.html
@@ -30,6 +30,9 @@
+ <link rel="index" title="Index"
+ href="../../genindex.html"/>
+ <link rel="search" title="Search" href="../../search.html"/>
<link rel="top" title="Airflow Documentation" href="../../index.html"/>
<link rel="up" title="Module code" href="../index.html"/>
@@ -40,6 +43,7 @@
<body class="wy-body-for-nav" role="document">
+
<div class="wy-grid-for-nav">
@@ -90,6 +94,8 @@
<li class="toctree-l1"><a class="reference internal" href="../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../integration.html">Integration</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../code.html">API Reference</a></li>
</ul>
@@ -104,8 +110,10 @@
<nav class="wy-nav-top" role="navigation" aria-label="top navigation">
- <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
- <a href="../../index.html">Airflow</a>
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../index.html">Airflow</a>
+
</nav>
@@ -118,19 +126,36 @@
+
+
+
+
+
+
+
+
+
+
<div role="navigation" aria-label="breadcrumbs navigation">
+
<ul class="wy-breadcrumbs">
- <li><a href="../../index.html">Docs</a> »</li>
-
+
+ <li><a href="../../index.html">Docs</a> »</li>
+
<li><a href="../index.html">Module code</a> »</li>
-
- <li>airflow.models</li>
+
+ <li>airflow.models</li>
+
+
<li class="wy-breadcrumbs-aside">
-
+
</li>
+
</ul>
+
+
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
@@ -150,24 +175,25 @@
<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
<span class="c1"># See the License for the specific language governing permissions and</span>
<span class="c1"># limitations under the License.</span>
-<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">absolute_import</span>
-<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">division</span>
-<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">print_function</span>
-<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">unicode_literals</span>
+<span class="kn">from</span> <span class="nn">__future__</span> <span class="k">import</span> <span class="n">absolute_import</span>
+<span class="kn">from</span> <span class="nn">__future__</span> <span class="k">import</span> <span class="n">division</span>
+<span class="kn">from</span> <span class="nn">__future__</span> <span class="k">import</span> <span class="n">print_function</span>
+<span class="kn">from</span> <span class="nn">__future__</span> <span class="k">import</span> <span class="n">unicode_literals</span>
-<span class="kn">from</span> <span class="nn">future.standard_library</span> <span class="kn">import</span> <span class="n">install_aliases</span>
+<span class="kn">from</span> <span class="nn">future.standard_library</span> <span class="k">import</span> <span class="n">install_aliases</span>
<span class="n">install_aliases</span><span class="p">()</span>
-<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
-<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">object</span><span class="p">,</span> <span class="nb">bytes</span>
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">str</span>
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">object</span><span class="p">,</span> <span class="nb">bytes</span>
<span class="kn">import</span> <span class="nn">copy</span>
-<span class="kn">from</span> <span class="nn">collections</span> <span class="kn">import</span> <span class="n">namedtuple</span>
-<span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span>
+<span class="kn">from</span> <span class="nn">collections</span> <span class="k">import</span> <span class="n">namedtuple</span>
+<span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span>
<span class="kn">import</span> <span class="nn">dill</span>
<span class="kn">import</span> <span class="nn">functools</span>
<span class="kn">import</span> <span class="nn">getpass</span>
<span class="kn">import</span> <span class="nn">imp</span>
<span class="kn">import</span> <span class="nn">importlib</span>
+<span class="kn">import</span> <span class="nn">inspect</span>
<span class="kn">import</span> <span class="nn">zipfile</span>
<span class="kn">import</span> <span class="nn">jinja2</span>
<span class="kn">import</span> <span class="nn">json</span>
@@ -181,66 +207,70 @@
<span class="kn">import</span> <span class="nn">textwrap</span>
<span class="kn">import</span> <span class="nn">traceback</span>
<span class="kn">import</span> <span class="nn">warnings</span>
-<span class="kn">from</span> <span class="nn">urllib.parse</span> <span class="kn">import</span> <span class="n">urlparse</span>
+<span class="kn">import</span> <span class="nn">hashlib</span>
-<span class="kn">from</span> <span class="nn">sqlalchemy</span> <span class="kn">import</span> <span class="p">(</span>
+<span class="kn">from</span> <span class="nn">urllib.parse</span> <span class="k">import</span> <span class="n">urlparse</span>
+
+<span class="kn">from</span> <span class="nn">sqlalchemy</span> <span class="k">import</span> <span class="p">(</span>
<span class="n">Column</span><span class="p">,</span> <span class="n">Integer</span><span class="p">,</span> <span class="n">String</span><span class="p">,</span> <span class="n">DateTime</span><span class="p">,</span> <span class="n">Text</span><span class="p">,</span> <span class="n">Boolean</span><span class="p">,</span> <span class="n">ForeignKey</span><span class="p">,</span> <span class="n">PickleType</span><span class="p">,</span>
<span class="n">Index</span><span class="p">,</span> <span class="n">Float</span><span class="p">)</span>
-<span class="kn">from</span> <span class="nn">sqlalchemy</span> <span class="kn">import</span> <span class="n">case</span><span class="p">,</span> <span class="n">func</span><span class="p">,</span> <span class="n">or_</span><span class="p">,</span> <span class="n">and_</span>
-<span class="kn">from</span> <span class="nn">sqlalchemy.ext.declarative</span> <span class="kn">import</span> <span class="n">declarative_base</span><span class="p">,</span> <span class="n">declared_attr</span>
-<span class="kn">from</span> <span class="nn">sqlalchemy.dialects.mysql</span> <span class="kn">import</span> <span class="n">LONGTEXT</span>
-<span class="kn">from</span> <span class="nn">sqlalchemy.orm</span> <span class="kn">import</span> <span class="n">relationship</span><span class="p">,</span> <span class="n">synonym</span>
+<span class="kn">from</span> <span class="nn">sqlalchemy</span> <span class="k">import</span> <span class="n">func</span><span class="p">,</span> <span class="n">or_</span><span class="p">,</span> <span class="n">and_</span>
+<span class="kn">from</span> <span class="nn">sqlalchemy.ext.declarative</span> <span class="k">import</span> <span class="n">declarative_base</span><span class="p">,</span> <span class="n">declared_attr</span>
+<span class="kn">from</span> <span class="nn">sqlalchemy.dialects.mysql</span> <span class="k">import</span> <span class="n">LONGTEXT</span>
+<span class="kn">from</span> <span class="nn">sqlalchemy.orm</span> <span class="k">import</span> <span class="n">reconstructor</span><span class="p">,</span> <span class="n">relationship</span><span class="p">,</span> <span class="n">synonym</span>
-<span class="kn">from</span> <span class="nn">croniter</span> <span class="kn">import</span> <span class="n">croniter</span>
+<span class="kn">from</span> <span class="nn">croniter</span> <span class="k">import</span> <span class="n">croniter</span>
<span class="kn">import</span> <span class="nn">six</span>
-<span class="kn">from</span> <span class="nn">airflow</span> <span class="kn">import</span> <span class="n">settings</span><span class="p">,</span> <span class="n">utils</span>
-<span class="kn">from</span> <span class="nn">airflow.executors</span> <span class="kn">import</span> <span class="n">DEFAULT_EXECUTOR</span><span class="p">,</span> <span class="n">LocalExecutor</span>
-<span class="kn">from</span> <span class="nn">airflow</span> <span class="kn">import</span> <span class="n">configuration</span>
-<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span><span class="p">,</span> <span class="n">AirflowSkipException</span>
-<span class="kn">from</span> <span class="nn">airflow.dag.base_dag</span> <span class="kn">import</span> <span class="n">BaseDag</span><span class="p">,</span> <span class="n">BaseDagBag</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.dates</span> <span class="kn">import</span> <span class="n">cron_presets</span><span class="p">,</span> <span class="n">date_range</span> <span class="k">as</span> <span class="n">utils_date_range</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.db</span> <span class="kn">import</span> <span class="n">provide_session</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.email</span> <span class="kn">import</span> <span class="n">send_email</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.helpers</span> <span class="kn">import</span> <span class="p">(</span>
+<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">settings</span><span class="p">,</span> <span class="n">utils</span>
+<span class="kn">from</span> <span class="nn">airflow.executors</span> <span class="k">import</span> <span class="n">DEFAULT_EXECUTOR</span><span class="p">,</span> <span class="n">LocalExecutor</span>
+<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">configuration</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span><span class="p">,</span> <span class="n">AirflowSkipException</span><span class="p">,</span> <span class="n">AirflowTaskTimeout</span>
+<span class="kn">from</span> <span class="nn">airflow.dag.base_dag</span> <span class="k">import</span> <span class="n">BaseDag</span><span class="p">,</span> <span class="n">BaseDagBag</span>
+<span class="kn">from</span> <span class="nn">airflow.ti_deps.deps.not_in_retry_period_dep</span> <span class="k">import</span> <span class="n">NotInRetryPeriodDep</span>
+<span class="kn">from</span> <span class="nn">airflow.ti_deps.deps.prev_dagrun_dep</span> <span class="k">import</span> <span class="n">PrevDagrunDep</span>
+<span class="kn">from</span> <span class="nn">airflow.ti_deps.deps.trigger_rule_dep</span> <span class="k">import</span> <span class="n">TriggerRuleDep</span>
+<span class="kn">from</span> <span class="nn">airflow.ti_deps.dep_context</span> <span class="k">import</span> <span class="n">DepContext</span><span class="p">,</span> <span class="n">QUEUE_DEPS</span><span class="p">,</span> <span class="n">RUN_DEPS</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.dates</span> <span class="k">import</span> <span class="n">cron_presets</span><span class="p">,</span> <span class="n">date_range</span> <span class="k">as</span> <span class="n">utils_date_range</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.db</span> <span class="k">import</span> <span class="n">provide_session</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.email</span> <span class="k">import</span> <span class="n">send_email</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.helpers</span> <span class="k">import</span> <span class="p">(</span>
<span class="n">as_tuple</span><span class="p">,</span> <span class="n">is_container</span><span class="p">,</span> <span class="n">is_in</span><span class="p">,</span> <span class="n">validate_key</span><span class="p">,</span> <span class="n">pprinttable</span><span class="p">)</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.logging</span> <span class="kn">import</span> <span class="n">LoggingMixin</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.operator_resources</span> <span class="kn">import</span> <span class="n">Resources</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="kn">import</span> <span class="n">State</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.timeout</span> <span class="kn">import</span> <span class="n">timeout</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.trigger_rule</span> <span class="kn">import</span> <span class="n">TriggerRule</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.logging</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.operator_resources</span> <span class="k">import</span> <span class="n">Resources</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="k">import</span> <span class="n">State</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.timeout</span> <span class="k">import</span> <span class="n">timeout</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.trigger_rule</span> <span class="k">import</span> <span class="n">TriggerRule</span>
<span class="n">Base</span> <span class="o">=</span> <span class="n">declarative_base</span><span class="p">()</span>
<span class="n">ID_LEN</span> <span class="o">=</span> <span class="mi">250</span>
-<span class="n">SQL_ALCHEMY_CONN</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'core'</span><span class="p">,</span> <span class="s1">'SQL_ALCHEMY_CONN'</span><span class="p">)</span>
-<span class="n">DAGS_FOLDER</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">expanduser</span><span class="p">(</span><span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'core'</span><span class="p">,</span> <span class="s1">'DAGS_FOLDER'</span><span class="p">))</span>
<span class="n">XCOM_RETURN_KEY</span> <span class="o">=</span> <span class="s1">'return_value'</span>
<span class="n">Stats</span> <span class="o">=</span> <span class="n">settings</span><span class="o">.</span><span class="n">Stats</span>
-<span class="n">ENCRYPTION_ON</span> <span class="o">=</span> <span class="bp">False</span>
+<span class="n">ENCRYPTION_ON</span> <span class="o">=</span> <span class="kc">False</span>
<span class="k">try</span><span class="p">:</span>
- <span class="kn">from</span> <span class="nn">cryptography.fernet</span> <span class="kn">import</span> <span class="n">Fernet</span>
+ <span class="kn">from</span> <span class="nn">cryptography.fernet</span> <span class="k">import</span> <span class="n">Fernet</span>
<span class="n">FERNET</span> <span class="o">=</span> <span class="n">Fernet</span><span class="p">(</span><span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'core'</span><span class="p">,</span> <span class="s1">'FERNET_KEY'</span><span class="p">)</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">))</span>
- <span class="n">ENCRYPTION_ON</span> <span class="o">=</span> <span class="bp">True</span>
+ <span class="n">ENCRYPTION_ON</span> <span class="o">=</span> <span class="kc">True</span>
<span class="k">except</span><span class="p">:</span>
<span class="k">pass</span>
-<span class="k">if</span> <span class="s1">'mysql'</span> <span class="ow">in</span> <span class="n">SQL_ALCHEMY_CONN</span><span class="p">:</span>
+<span class="k">if</span> <span class="s1">'mysql'</span> <span class="ow">in</span> <span class="n">settings</span><span class="o">.</span><span class="n">SQL_ALCHEMY_CONN</span><span class="p">:</span>
<span class="n">LongText</span> <span class="o">=</span> <span class="n">LONGTEXT</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">LongText</span> <span class="o">=</span> <span class="n">Text</span>
<span class="c1"># used by DAG context_managers</span>
-<span class="n">_CONTEXT_MANAGER_DAG</span> <span class="o">=</span> <span class="bp">None</span>
+<span class="n">_CONTEXT_MANAGER_DAG</span> <span class="o">=</span> <span class="kc">None</span>
-<span class="k">def</span> <span class="nf">clear_task_instances</span><span class="p">(</span><span class="n">tis</span><span class="p">,</span> <span class="n">session</span><span class="p">,</span> <span class="n">activate_dag_runs</span><span class="o">=</span><span class="bp">True</span><span class="p">):</span>
- <span class="sd">'''</span>
+<span class="k">def</span> <span class="nf">clear_task_instances</span><span class="p">(</span><span class="n">tis</span><span class="p">,</span> <span class="n">session</span><span class="p">,</span> <span class="n">activate_dag_runs</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
+ <span class="sd">"""</span>
<span class="sd"> Clears a set of task instances, but makes sure the running ones</span>
<span class="sd"> get killed.</span>
-<span class="sd"> '''</span>
+<span class="sd"> """</span>
<span class="n">job_ids</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">ti</span> <span class="ow">in</span> <span class="n">tis</span><span class="p">:</span>
<span class="k">if</span> <span class="n">ti</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">State</span><span class="o">.</span><span class="n">RUNNING</span><span class="p">:</span>
@@ -248,13 +278,13 @@
<span class="n">ti</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="n">State</span><span class="o">.</span><span class="n">SHUTDOWN</span>
<span class="n">job_ids</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">ti</span><span class="o">.</span><span class="n">job_id</span><span class="p">)</span>
<span class="c1"># todo: this creates an issue with the webui tests</span>
- <span class="c1">#elif ti.state != State.REMOVED:</span>
- <span class="c1"># ti.state = State.NONE</span>
- <span class="c1"># session.merge(ti)</span>
+ <span class="c1"># elif ti.state != State.REMOVED:</span>
+ <span class="c1"># ti.state = State.NONE</span>
+ <span class="c1"># session.merge(ti)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">session</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span><span class="n">ti</span><span class="p">)</span>
<span class="k">if</span> <span class="n">job_ids</span><span class="p">:</span>
- <span class="kn">from</span> <span class="nn">airflow.jobs</span> <span class="kn">import</span> <span class="n">BaseJob</span> <span class="k">as</span> <span class="n">BJ</span>
+ <span class="kn">from</span> <span class="nn">airflow.jobs</span> <span class="k">import</span> <span class="n">BaseJob</span> <span class="k">as</span> <span class="n">BJ</span>
<span class="k">for</span> <span class="n">job</span> <span class="ow">in</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">BJ</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">BJ</span><span class="o">.</span><span class="n">id</span><span class="o">.</span><span class="n">in_</span><span class="p">(</span><span class="n">job_ids</span><span class="p">))</span><span class="o">.</span><span class="n">all</span><span class="p">():</span>
<span class="n">job</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="n">State</span><span class="o">.</span><span class="n">SHUTDOWN</span>
<span class="k">if</span> <span class="n">activate_dag_runs</span><span class="p">:</span>
@@ -293,12 +323,12 @@
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
<span class="bp">self</span><span class="p">,</span>
- <span class="n">dag_folder</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+ <span class="n">dag_folder</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">executor</span><span class="o">=</span><span class="n">DEFAULT_EXECUTOR</span><span class="p">,</span>
<span class="n">include_examples</span><span class="o">=</span><span class="n">configuration</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'core'</span><span class="p">,</span> <span class="s1">'LOAD_EXAMPLES'</span><span class="p">)):</span>
- <span class="n">dag_folder</span> <span class="o">=</span> <span class="n">dag_folder</span> <span class="ow">or</span> <span class="n">DAGS_FOLDER</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Filling up the DagBag from {}"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">dag_folder</span><span class="p">))</span>
+ <span class="n">dag_folder</span> <span class="o">=</span> <span class="n">dag_folder</span> <span class="ow">or</span> <span class="n">settings</span><span class="o">.</span><span class="n">DAGS_FOLDER</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Filling up the DagBag from </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">dag_folder</span><span class="p">))</span>
<span class="bp">self</span><span class="o">.</span><span class="n">dag_folder</span> <span class="o">=</span> <span class="n">dag_folder</span>
<span class="bp">self</span><span class="o">.</span><span class="n">dags</span> <span class="o">=</span> <span class="p">{}</span>
<span class="c1"># the file's last modified timestamp when we last read it</span>
@@ -341,7 +371,7 @@
<span class="p">):</span>
<span class="c1"># Reprocess source file</span>
<span class="n">found_dags</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">process_file</span><span class="p">(</span>
- <span class="n">filepath</span><span class="o">=</span><span class="n">orm_dag</span><span class="o">.</span><span class="n">fileloc</span><span class="p">,</span> <span class="n">only_if_updated</span><span class="o">=</span><span class="bp">False</span><span class="p">)</span>
+ <span class="n">filepath</span><span class="o">=</span><span class="n">orm_dag</span><span class="o">.</span><span class="n">fileloc</span><span class="p">,</span> <span class="n">only_if_updated</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
<span class="c1"># If the source file no longer exports `dag_id`, delete it from self.dags</span>
<span class="k">if</span> <span class="n">found_dags</span> <span class="ow">and</span> <span class="n">dag_id</span> <span class="ow">in</span> <span class="p">[</span><span class="n">dag</span><span class="o">.</span><span class="n">dag_id</span> <span class="k">for</span> <span class="n">dag</span> <span class="ow">in</span> <span class="n">found_dags</span><span class="p">]:</span>
@@ -350,7 +380,7 @@
<span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">dags</span><span class="p">[</span><span class="n">dag_id</span><span class="p">]</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">dags</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">dag_id</span><span class="p">)</span></div>
-<div class="viewcode-block" id="DagBag.process_file"><a class="viewcode-back" href="../../code.html#airflow.models.DagBag.process_file">[docs]</a> <span class="k">def</span> <span class="nf">process_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">filepath</span><span class="p">,</span> <span class="n">only_if_updated</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span> <span class="n">safe_mode</span><span class="o">=</span><span class="bp">True</span><span class="p">):</span>
+<div class="viewcode-block" id="DagBag.process_file"><a class="viewcode-back" href="../../code.html#airflow.models.DagBag.process_file">[docs]</a> <span class="k">def</span> <span class="nf">process_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">filepath</span><span class="p">,</span> <span class="n">only_if_updated</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">safe_mode</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Given a path to a python module or zip file, this method imports</span>
<span class="sd"> the module and look for dag objects within it.</span>
@@ -380,11 +410,14 @@
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">filepath</span><span class="p">,</span> <span class="s1">'rb'</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
<span class="n">content</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">()</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">all</span><span class="p">([</span><span class="n">s</span> <span class="ow">in</span> <span class="n">content</span> <span class="k">for</span> <span class="n">s</span> <span class="ow">in</span> <span class="p">(</span><span class="n">b</span><span class="s1">'DAG'</span><span class="p">,</span> <span class="n">b</span><span class="s1">'airflow'</span><span class="p">)]):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">file_last_changed</span><span class="p">[</span><span class="n">filepath</span><span class="p">]</span> <span class="o">=</span> <span class="n">file_last_changed_on_disk</span>
<span class="k">return</span> <span class="n">found_dags</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Importing {}"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">filepath</span><span class="p">))</span>
- <span class="n">org_mod_name</span><span class="p">,</span> <span class="n">file_ext</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">splitext</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="n">filepath</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span>
- <span class="n">mod_name</span> <span class="o">=</span> <span class="s1">'unusual_prefix_'</span> <span class="o">+</span> <span class="n">org_mod_name</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Importing </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">filepath</span><span class="p">))</span>
+ <span class="n">org_mod_name</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">splitext</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="n">filepath</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span>
+ <span class="n">mod_name</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'unusual_prefix_'</span> <span class="o">+</span>
+ <span class="n">hashlib</span><span class="o">.</span><span class="n">sha1</span><span class="p">(</span><span class="n">filepath</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">))</span><span class="o">.</span><span class="n">hexdigest</span><span class="p">()</span> <span class="o">+</span>
+ <span class="s1">'_'</span> <span class="o">+</span> <span class="n">org_mod_name</span><span class="p">)</span>
<span class="k">if</span> <span class="n">mod_name</span> <span class="ow">in</span> <span class="n">sys</span><span class="o">.</span><span class="n">modules</span><span class="p">:</span>
<span class="k">del</span> <span class="n">sys</span><span class="o">.</span><span class="n">modules</span><span class="p">[</span><span class="n">mod_name</span><span class="p">]</span>
@@ -401,19 +434,21 @@
<span class="k">else</span><span class="p">:</span>
<span class="n">zip_file</span> <span class="o">=</span> <span class="n">zipfile</span><span class="o">.</span><span class="n">ZipFile</span><span class="p">(</span><span class="n">filepath</span><span class="p">)</span>
<span class="k">for</span> <span class="n">mod</span> <span class="ow">in</span> <span class="n">zip_file</span><span class="o">.</span><span class="n">infolist</span><span class="p">():</span>
- <span class="n">head</span><span class="p">,</span> <span class="n">tail</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="n">mod</span><span class="o">.</span><span class="n">filename</span><span class="p">)</span>
+ <span class="n">head</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="n">mod</span><span class="o">.</span><span class="n">filename</span><span class="p">)</span>
<span class="n">mod_name</span><span class="p">,</span> <span class="n">ext</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">splitext</span><span class="p">(</span><span class="n">mod</span><span class="o">.</span><span class="n">filename</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">head</span> <span class="ow">and</span> <span class="p">(</span><span class="n">ext</span> <span class="o">==</span> <span class="s1">'.py'</span> <span class="ow">or</span> <span class="n">ext</span> <span class="o">==</span> <span class="s1">'.pyc'</span><span class="p">):</span>
<span class="k">if</span> <span class="n">mod_name</span> <span class="o">==</span> <span class="s1">'__init__'</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"Found __init__.{0} at root of {1}"</span><span class="o">.</span>
- <span class="n">format</span><span class="p">(</span><span class="n">ext</span><span class="p">,</span> <span class="n">filepath</span><span class="p">))</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"Found __init__.</span><span class="si">{0}</span><span class="s2"> at root of </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span>
+ <span class="nb">format</span><span class="p">(</span><span class="n">ext</span><span class="p">,</span> <span class="n">filepath</span><span class="p">))</span>
<span class="k">if</span> <span class="n">safe_mode</span><span class="p">:</span>
<span class="k">with</span> <span class="n">zip_file</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">mod</span><span class="o">.</span><span class="n">filename</span><span class="p">)</span> <span class="k">as</span> <span class="n">zf</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Reading {} from {}"</span><span class="o">.</span>
- <span class="n">format</span><span class="p">(</span><span class="n">mod</span><span class="o">.</span><span class="n">filename</span><span class="p">,</span> <span class="n">filepath</span><span class="p">))</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Reading </span><span class="si">{}</span><span class="s2"> from </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span>
+ <span class="nb">format</span><span class="p">(</span><span class="n">mod</span><span class="o">.</span><span class="n">filename</span><span class="p">,</span> <span class="n">filepath</span><span class="p">))</span>
<span class="n">content</span> <span class="o">=</span> <span class="n">zf</span><span class="o">.</span><span class="n">read</span><span class="p">()</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">all</span><span class="p">([</span><span class="n">s</span> <span class="ow">in</span> <span class="n">content</span> <span class="k">for</span> <span class="n">s</span> <span class="ow">in</span> <span class="p">(</span><span class="n">b</span><span class="s1">'DAG'</span><span class="p">,</span> <span class="n">b</span><span class="s1">'airflow'</span><span class="p">)]):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">file_last_changed</span><span class="p">[</span><span class="n">filepath</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span>
+ <span class="n">file_last_changed_on_disk</span><span class="p">)</span>
<span class="c1"># todo: create ignore list</span>
<span class="k">return</span> <span class="n">found_dags</span>
@@ -434,8 +469,7 @@
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">dag</span><span class="p">,</span> <span class="n">DAG</span><span class="p">):</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">dag</span><span class="o">.</span><span class="n">full_filepath</span><span class="p">:</span>
<span class="n">dag</span><span class="o">.</span><span class="n">full_filepath</span> <span class="o">=</span> <span class="n">filepath</span>
- <span class="n">dag</span><span class="o">.</span><span class="n">is_subdag</span> <span class="o">=</span> <span class="bp">False</span>
- <span class="n">dag</span><span class="o">.</span><span class="n">module_name</span> <span class="o">=</span> <span class="n">m</span><span class="o">.</span><span class="n">__name__</span>
+ <span class="n">dag</span><span class="o">.</span><span class="n">is_subdag</span> <span class="o">=</span> <span class="kc">False</span>
<span class="bp">self</span><span class="o">.</span><span class="n">bag_dag</span><span class="p">(</span><span class="n">dag</span><span class="p">,</span> <span class="n">parent_dag</span><span class="o">=</span><span class="n">dag</span><span class="p">,</span> <span class="n">root_dag</span><span class="o">=</span><span class="n">dag</span><span class="p">)</span>
<span class="n">found_dags</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">dag</span><span class="p">)</span>
<span class="n">found_dags</span> <span class="o">+=</span> <span class="n">dag</span><span class="o">.</span><span class="n">subdags</span>
@@ -444,18 +478,18 @@
<span class="k">return</span> <span class="n">found_dags</span></div>
<span class="nd">@provide_session</span>
-<div class="viewcode-block" id="DagBag.kill_zombies"><a class="viewcode-back" href="../../code.html#airflow.models.DagBag.kill_zombies">[docs]</a> <span class="k">def</span> <span class="nf">kill_zombies</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
+<div class="viewcode-block" id="DagBag.kill_zombies"><a class="viewcode-back" href="../../code.html#airflow.models.DagBag.kill_zombies">[docs]</a> <span class="k">def</span> <span class="nf">kill_zombies</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Fails tasks that haven't had a heartbeat in too long</span>
<span class="sd"> """</span>
- <span class="kn">from</span> <span class="nn">airflow.jobs</span> <span class="kn">import</span> <span class="n">LocalTaskJob</span> <span class="k">as</span> <span class="n">LJ</span>
+ <span class="kn">from</span> <span class="nn">airflow.jobs</span> <span class="k">import</span> <span class="n">LocalTaskJob</span> <span class="k">as</span> <span class="n">LJ</span>
<span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Finding 'running' jobs without a recent heartbeat"</span><span class="p">)</span>
<span class="n">TI</span> <span class="o">=</span> <span class="n">TaskInstance</span>
<span class="n">secs</span> <span class="o">=</span> <span class="p">(</span>
- <span class="n">configuration</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">'scheduler'</span><span class="p">,</span> <span class="s1">'job_heartbeat_sec'</span><span class="p">)</span> <span class="o">*</span> <span class="mi">3</span><span class="p">)</span> <span class="o">+</span> <span class="mi">120</span>
+ <span class="n">configuration</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">'scheduler'</span><span class="p">,</span> <span class="s1">'scheduler_zombie_task_threshold'</span><span class="p">))</span>
<span class="n">limit_dttm</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> <span class="o">-</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">seconds</span><span class="o">=</span><span class="n">secs</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
- <span class="s2">"Failing jobs without heartbeat after {}"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">limit_dttm</span><span class="p">))</span>
+ <span class="s2">"Failing jobs without heartbeat after </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">limit_dttm</span><span class="p">))</span>
<span class="n">tis</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">TI</span><span class="p">)</span>
@@ -475,9 +509,9 @@
<span class="k">if</span> <span class="n">ti</span><span class="o">.</span><span class="n">task_id</span> <span class="ow">in</span> <span class="n">dag</span><span class="o">.</span><span class="n">task_ids</span><span class="p">:</span>
<span class="n">task</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">get_task</span><span class="p">(</span><span class="n">ti</span><span class="o">.</span><span class="n">task_id</span><span class="p">)</span>
<span class="n">ti</span><span class="o">.</span><span class="n">task</span> <span class="o">=</span> <span class="n">task</span>
- <span class="n">ti</span><span class="o">.</span><span class="n">handle_failure</span><span class="p">(</span><span class="s2">"{} killed as zombie"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">ti</span><span class="p">))</span>
+ <span class="n">ti</span><span class="o">.</span><span class="n">handle_failure</span><span class="p">(</span><span class="s2">"</span><span class="si">{}</span><span class="s2"> killed as zombie"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">ti</span><span class="p">))</span>
<span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
- <span class="s1">'Marked zombie job {} as failed'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">ti</span><span class="p">))</span>
+ <span class="s1">'Marked zombie job </span><span class="si">{}</span><span class="s1"> as failed'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">ti</span><span class="p">))</span>
<span class="n">Stats</span><span class="o">.</span><span class="n">incr</span><span class="p">(</span><span class="s1">'zombies_killed'</span><span class="p">)</span>
<span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span></div>
@@ -495,15 +529,14 @@
<span class="k">for</span> <span class="n">subdag</span> <span class="ow">in</span> <span class="n">dag</span><span class="o">.</span><span class="n">subdags</span><span class="p">:</span>
<span class="n">subdag</span><span class="o">.</span><span class="n">full_filepath</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">full_filepath</span>
<span class="n">subdag</span><span class="o">.</span><span class="n">parent_dag</span> <span class="o">=</span> <span class="n">dag</span>
- <span class="n">subdag</span><span class="o">.</span><span class="n">fileloc</span> <span class="o">=</span> <span class="n">root_dag</span><span class="o">.</span><span class="n">full_filepath</span>
- <span class="n">subdag</span><span class="o">.</span><span class="n">is_subdag</span> <span class="o">=</span> <span class="bp">True</span>
+ <span class="n">subdag</span><span class="o">.</span><span class="n">is_subdag</span> <span class="o">=</span> <span class="kc">True</span>
<span class="bp">self</span><span class="o">.</span><span class="n">bag_dag</span><span class="p">(</span><span class="n">subdag</span><span class="p">,</span> <span class="n">parent_dag</span><span class="o">=</span><span class="n">dag</span><span class="p">,</span> <span class="n">root_dag</span><span class="o">=</span><span class="n">root_dag</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'Loaded DAG {dag}'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span></div>
+ <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'Loaded DAG </span><span class="si">{dag}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span></div>
<div class="viewcode-block" id="DagBag.collect_dags"><a class="viewcode-back" href="../../code.html#airflow.models.DagBag.collect_dags">[docs]</a> <span class="k">def</span> <span class="nf">collect_dags</span><span class="p">(</span>
<span class="bp">self</span><span class="p">,</span>
- <span class="n">dag_folder</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
- <span class="n">only_if_updated</span><span class="o">=</span><span class="bp">True</span><span class="p">):</span>
+ <span class="n">dag_folder</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">only_if_updated</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Given a file path or a folder, this method looks for python modules,</span>
<span class="sd"> imports them and adds them to the dagbag collection.</span>
@@ -524,7 +557,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">process_file</span><span class="p">(</span><span class="n">dag_folder</span><span class="p">,</span> <span class="n">only_if_updated</span><span class="o">=</span><span class="n">only_if_updated</span><span class="p">)</span>
<span class="k">elif</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isdir</span><span class="p">(</span><span class="n">dag_folder</span><span class="p">):</span>
<span class="n">patterns</span> <span class="o">=</span> <span class="p">[]</span>
- <span class="k">for</span> <span class="n">root</span><span class="p">,</span> <span class="n">dirs</span><span class="p">,</span> <span class="n">files</span> <span class="ow">in</span> <span class="n">os</span><span class="o">.</span><span class="n">walk</span><span class="p">(</span><span class="n">dag_folder</span><span class="p">,</span> <span class="n">followlinks</span><span class="o">=</span><span class="bp">True</span><span class="p">):</span>
+ <span class="k">for</span> <span class="n">root</span><span class="p">,</span> <span class="n">dirs</span><span class="p">,</span> <span class="n">files</span> <span class="ow">in</span> <span class="n">os</span><span class="o">.</span><span class="n">walk</span><span class="p">(</span><span class="n">dag_folder</span><span class="p">,</span> <span class="n">followlinks</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="n">ignore_file</span> <span class="o">=</span> <span class="p">[</span><span class="n">f</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="n">files</span> <span class="k">if</span> <span class="n">f</span> <span class="o">==</span> <span class="s1">'.airflowignore'</span><span class="p">]</span>
<span class="k">if</span> <span class="n">ignore_file</span><span class="p">:</span>
<span class="n">f</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">root</span><span class="p">,</span> <span class="n">ignore_file</span><span class="p">[</span><span class="mi">0</span><span class="p">]),</span> <span class="s1">'r'</span><span class="p">)</span>
@@ -564,18 +597,18 @@
<span class="n">Stats</span><span class="o">.</span><span class="n">gauge</span><span class="p">(</span>
<span class="s1">'dagbag_import_errors'</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">import_errors</span><span class="p">),</span> <span class="mi">1</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">dagbag_stats</span> <span class="o">=</span> <span class="nb">sorted</span><span class="p">(</span>
- <span class="n">stats</span><span class="p">,</span> <span class="n">key</span><span class="o">=</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span><span class="o">.</span><span class="n">duration</span><span class="p">,</span> <span class="n">reverse</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span></div>
+ <span class="n">stats</span><span class="p">,</span> <span class="n">key</span><span class="o">=</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span><span class="o">.</span><span class="n">duration</span><span class="p">,</span> <span class="n">reverse</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span></div>
<div class="viewcode-block" id="DagBag.dagbag_report"><a class="viewcode-back" href="../../code.html#airflow.models.DagBag.dagbag_report">[docs]</a> <span class="k">def</span> <span class="nf">dagbag_report</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""Prints a report around DagBag loading stats"""</span>
<span class="n">report</span> <span class="o">=</span> <span class="n">textwrap</span><span class="o">.</span><span class="n">dedent</span><span class="p">(</span><span class="s2">"""</span><span class="se">\n</span><span class="s2"></span>
<span class="s2"> -------------------------------------------------------------------</span>
-<span class="s2"> DagBag loading stats for {dag_folder}</span>
+<span class="s2"> DagBag loading stats for </span><span class="si">{dag_folder}</span><span class="s2"></span>
<span class="s2"> -------------------------------------------------------------------</span>
-<span class="s2"> Number of DAGs: {dag_num}</span>
-<span class="s2"> Total task number: {task_num}</span>
-<span class="s2"> DagBag parsing time: {duration}</span>
-<span class="s2"> {table}</span>
+<span class="s2"> Number of DAGs: </span><span class="si">{dag_num}</span><span class="s2"></span>
+<span class="s2"> Total task number: </span><span class="si">{task_num}</span><span class="s2"></span>
+<span class="s2"> DagBag parsing time: </span><span class="si">{duration}</span><span class="s2"></span>
+<span class="s2"> </span><span class="si">{table}</span><span class="s2"></span>
<span class="s2"> """</span><span class="p">)</span>
<span class="n">stats</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">dagbag_stats</span>
<span class="k">return</span> <span class="n">report</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
@@ -591,7 +624,7 @@
<span class="n">session</span> <span class="o">=</span> <span class="n">settings</span><span class="o">.</span><span class="n">Session</span><span class="p">()</span>
<span class="k">for</span> <span class="n">dag</span> <span class="ow">in</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span>
<span class="n">DagModel</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="o">~</span><span class="n">DagModel</span><span class="o">.</span><span class="n">dag_id</span><span class="o">.</span><span class="n">in_</span><span class="p">(</span><span class="n">active_dag_ids</span><span class="p">))</span><span class="o">.</span><span class="n">all</span><span class="p">():</span>
- <span class="n">dag</span><span class="o">.</span><span class="n">is_active</span> <span class="o">=</span> <span class="bp">False</span>
+ <span class="n">dag</span><span class="o">.</span><span class="n">is_active</span> <span class="o">=</span> <span class="kc">False</span>
<span class="n">session</span><span class="o">.</span><span class="n">merge</span><span class="p">(</span><span class="n">dag</span><span class="p">)</span>
<span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
<span class="n">session</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
@@ -599,7 +632,7 @@
<span class="k">def</span> <span class="nf">paused_dags</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="n">session</span> <span class="o">=</span> <span class="n">settings</span><span class="o">.</span><span class="n">Session</span><span class="p">()</span>
<span class="n">dag_ids</span> <span class="o">=</span> <span class="p">[</span><span class="n">dp</span><span class="o">.</span><span class="n">dag_id</span> <span class="k">for</span> <span class="n">dp</span> <span class="ow">in</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DagModel</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span>
- <span class="n">DagModel</span><span class="o">.</span><span class="n">is_paused</span> <span class="o">==</span> <span class="bp">True</span><span class="p">)]</span>
+ <span class="n">DagModel</span><span class="o">.</span><span class="n">is_paused</span><span class="o">.</span><span class="n">__eq__</span><span class="p">(</span><span class="kc">True</span><span class="p">))]</span>
<span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
<span class="n">session</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
<span class="k">return</span> <span class="n">dag_ids</span></div>
@@ -608,10 +641,10 @@
<span class="k">class</span> <span class="nc">User</span><span class="p">(</span><span class="n">Base</span><span class="p">):</span>
<span class="n">__tablename__</span> <span class="o">=</span> <span class="s2">"users"</span>
- <span class="nb">id</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Integer</span><span class="p">,</span> <span class="n">primary_key</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span>
- <span class="n">username</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="n">ID_LEN</span><span class="p">),</span> <span class="n">unique</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span>
+ <span class="nb">id</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Integer</span><span class="p">,</span> <span class="n">primary_key</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+ <span class="n">username</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="n">ID_LEN</span><span class="p">),</span> <span class="n">unique</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="n">email</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="mi">500</span><span class="p">))</span>
- <span class="n">superuser</span> <span class="o">=</span> <span class="bp">False</span>
+ <span class="n">superuser</span> <span class="o">=</span> <span class="kc">False</span>
<span class="k">def</span> <span class="nf">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">username</span>
@@ -632,7 +665,7 @@
<span class="sd"> """</span>
<span class="n">__tablename__</span> <span class="o">=</span> <span class="s2">"connection"</span>
- <span class="nb">id</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Integer</span><span class="p">(),</span> <span class="n">primary_key</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span>
+ <span class="nb">id</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Integer</span><span class="p">(),</span> <span class="n">primary_key</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="n">conn_id</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="n">ID_LEN</span><span class="p">))</span>
<span class="n">conn_type</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="mi">500</span><span class="p">))</span>
<span class="n">host</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="mi">500</span><span class="p">))</span>
@@ -640,15 +673,40 @@
<span class="n">login</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="mi">500</span><span class="p">))</span>
<span class="n">_password</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="s1">'password'</span><span class="p">,</span> <span class="n">String</span><span class="p">(</span><span class="mi">5000</span><span class="p">))</span>
<span class="n">port</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Integer</span><span class="p">())</span>
- <span class="n">is_encrypted</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Boolean</span><span class="p">,</span> <span class="n">unique</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="bp">False</span><span class="p">)</span>
- <span class="n">is_extra_encrypted</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Boolean</span><span class="p">,</span> <span class="n">unique</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="bp">False</span><span class="p">)</span>
+ <span class="n">is_encrypted</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Boolean</span><span class="p">,</span> <span class="n">unique</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
+ <span class="n">is_extra_encrypted</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Boolean</span><span class="p">,</span> <span class="n">unique</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
<span class="n">_extra</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="s1">'extra'</span><span class="p">,</span> <span class="n">String</span><span class="p">(</span><span class="mi">5000</span><span class="p">))</span>
+ <span class="n">_types</span> <span class="o">=</span> <span class="p">[</span>
+ <span class="p">(</span><span class="s1">'fs'</span><span class="p">,</span> <span class="s1">'File (path)'</span><span class="p">),</span>
+ <span class="p">(</span><span class="s1">'ftp'</span><span class="p">,</span> <span class="s1">'FTP'</span><span class="p">,),</span>
+ <span class="p">(</span><span class="s1">'google_cloud_platform'</span><span class="p">,</span> <span class="s1">'Google Cloud Platform'</span><span class="p">),</span>
+ <span class="p">(</span><span class="s1">'hdfs'</span><span class="p">,</span> <span class="s1">'HDFS'</span><span class="p">,),</span>
+ <span class="p">(</span><span class="s1">'http'</span><span class="p">,</span> <span class="s1">'HTTP'</span><span class="p">,),</span>
+ <span class="p">(</span><span class="s1">'hive_cli'</span><span class="p">,</span> <span class="s1">'Hive Client Wrapper'</span><span class="p">,),</span>
+ <span class="p">(</span><span class="s1">'hive_metastore'</span><span class="p">,</span> <span class="s1">'Hive Metastore Thrift'</span><span class="p">,),</span>
+ <span class="p">(</span><span class="s1">'hiveserver2'</span><span class="p">,</span> <span class="s1">'Hive Server 2 Thrift'</span><span class="p">,),</span>
+ <span class="p">(</span><span class="s1">'jdbc'</span><span class="p">,</span> <span class="s1">'Jdbc Connection'</span><span class="p">,),</span>
+ <span class="p">(</span><span class="s1">'mysql'</span><span class="p">,</span> <span class="s1">'MySQL'</span><span class="p">,),</span>
+ <span class="p">(</span><span class="s1">'postgres'</span><span class="p">,</span> <span class="s1">'Postgres'</span><span class="p">,),</span>
+ <span class="p">(</span><span class="s1">'oracle'</span><span class="p">,</span> <span class="s1">'Oracle'</span><span class="p">,),</span>
+ <span class="p">(</span><span class="s1">'vertica'</span><span class="p">,</span> <span class="s1">'Vertica'</span><span class="p">,),</span>
+ <span class="p">(</span><span class="s1">'presto'</span><span class="p">,</span> <span class="s1">'Presto'</span><span class="p">,),</span>
+ <span class="p">(</span><span class="s1">'s3'</span><span class="p">,</span> <span class="s1">'S3'</span><span class="p">,),</span>
+ <span class="p">(</span><span class="s1">'samba'</span><span class="p">,</span> <span class="s1">'Samba'</span><span class="p">,),</span>
+ <span class="p">(</span><span class="s1">'sqlite'</span><span class="p">,</span> <span class="s1">'Sqlite'</span><span class="p">,),</span>
+ <span class="p">(</span><span class="s1">'ssh'</span><span class="p">,</span> <span class="s1">'SSH'</span><span class="p">,),</span>
+ <span class="p">(</span><span class="s1">'cloudant'</span><span class="p">,</span> <span class="s1">'IBM Cloudant'</span><span class="p">,),</span>
+ <span class="p">(</span><span class="s1">'mssql'</span><span class="p">,</span> <span class="s1">'Microsoft SQL Server'</span><span class="p">),</span>
+ <span class="p">(</span><span class="s1">'mesos_framework-id'</span><span class="p">,</span> <span class="s1">'Mesos Framework ID'</span><span class="p">),</span>
+ <span class="p">(</span><span class="s1">'jira'</span><span class="p">,</span> <span class="s1">'JIRA'</span><span class="p">,),</span>
+ <span class="p">]</span>
+
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
- <span class="bp">self</span><span class="p">,</span> <span class="n">conn_id</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">conn_type</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
- <span class="n">host</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">login</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">password</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
- <span class="n">schema</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">port</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">extra</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
- <span class="n">uri</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
+ <span class="bp">self</span><span class="p">,</span> <span class="n">conn_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">conn_type</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">host</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">login</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">password</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">schema</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">port</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">extra</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">uri</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span> <span class="o">=</span> <span class="n">conn_id</span>
<span class="k">if</span> <span class="n">uri</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">parse_from_uri</span><span class="p">(</span><span class="n">uri</span><span class="p">)</span>
@@ -680,7 +738,7 @@
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_password</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_encrypted</span><span class="p">:</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">ENCRYPTION_ON</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
- <span class="s2">"Can't decrypt encrypted password for login={}, </span><span class="se">\</span>
+ <span class="s2">"Can't decrypt encrypted password for login=</span><span class="si">{}</span><span class="s2">, </span><span class="se">\</span>
<span class="s2"> FERNET_KEY configuration is missing"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">login</span><span class="p">))</span>
<span class="k">return</span> <span class="n">FERNET</span><span class="o">.</span><span class="n">decrypt</span><span class="p">(</span><span class="nb">bytes</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_password</span><span class="p">,</span> <span class="s1">'utf-8'</span><span class="p">))</span><span class="o">.</span><span class="n">decode</span><span class="p">()</span>
<span class="k">else</span><span class="p">:</span>
@@ -690,10 +748,10 @@
<span class="k">if</span> <span class="n">value</span><span class="p">:</span>
<span class="k">try</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_password</span> <span class="o">=</span> <span class="n">FERNET</span><span class="o">.</span><span class="n">encrypt</span><span class="p">(</span><span class="nb">bytes</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="s1">'utf-8'</span><span class="p">))</span><span class="o">.</span><span class="n">decode</span><span class="p">()</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">is_encrypted</span> <span class="o">=</span> <span class="bp">True</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">is_encrypted</span> <span class="o">=</span> <span class="kc">True</span>
<span class="k">except</span> <span class="ne">NameError</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_password</span> <span class="o">=</span> <span class="n">value</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">is_encrypted</span> <span class="o">=</span> <span class="bp">False</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">is_encrypted</span> <span class="o">=</span> <span class="kc">False</span>
<span class="nd">@declared_attr</span>
<span class="k">def</span> <span class="nf">password</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
@@ -704,7 +762,7 @@
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_extra</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_extra_encrypted</span><span class="p">:</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">ENCRYPTION_ON</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
- <span class="s2">"Can't decrypt `extra` params for login={},</span><span class="se">\</span>
+ <span class="s2">"Can't decrypt `extra` params for login=</span><span class="si">{}</span><span class="s2">,</span><span class="se">\</span>
<span class="s2"> FERNET_KEY configuration is missing"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">login</span><span class="p">))</span>
<span class="k">return</span> <span class="n">FERNET</span><span class="o">.</span><span class="n">decrypt</span><span class="p">(</span><span class="nb">bytes</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_extra</span><span class="p">,</span> <span class="s1">'utf-8'</span><span class="p">))</span><span class="o">.</span><span class="n">decode</span><span class="p">()</span>
<span class="k">else</span><span class="p">:</span>
@@ -714,10 +772,10 @@
<span class="k">if</span> <span class="n">value</span><span class="p">:</span>
<span class="k">try</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_extra</span> <span class="o">=</span> <span class="n">FERNET</span><span class="o">.</span><span class="n">encrypt</span><span class="p">(</span><span class="nb">bytes</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="s1">'utf-8'</span><span class="p">))</span><span class="o">.</span><span class="n">decode</span><span class="p">()</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">is_extra_encrypted</span> <span class="o">=</span> <span class="bp">True</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">is_extra_encrypted</span> <span class="o">=</span> <span class="kc">True</span>
<span class="k">except</span> <span class="ne">NameError</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_extra</span> <span class="o">=</span> <span class="n">value</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">is_extra_encrypted</span> <span class="o">=</span> <span class="bp">False</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">is_extra_encrypted</span> <span class="o">=</span> <span class="kc">False</span>
<span class="nd">@declared_attr</span>
<span class="k">def</span> <span class="nf">extra</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
@@ -727,59 +785,61 @@
<span class="k">def</span> <span class="nf">get_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">try</span><span class="p">:</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">'mysql'</span><span class="p">:</span>
- <span class="kn">from</span> <span class="nn">airflow.hooks.mysql_hook</span> <span class="kn">import</span> <span class="n">MySqlHook</span>
+ <span class="kn">from</span> <span class="nn">airflow.hooks.mysql_hook</span> <span class="k">import</span> <span class="n">MySqlHook</span>
<span class="k">return</span> <span class="n">MySqlHook</span><span class="p">(</span><span class="n">mysql_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
<span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">'google_cloud_platform'</span><span class="p">:</span>
- <span class="kn">from</span> <span class="nn">airflow.contrib.hooks.bigquery_hook</span> <span class="kn">import</span> <span class="n">BigQueryHook</span>
+ <span class="kn">from</span> <span class="nn">airflow.contrib.hooks.bigquery_hook</span> <span class="k">import</span> <span class="n">BigQueryHook</span>
<span class="k">return</span> <span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
<span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">'postgres'</span><span class="p">:</span>
- <span class="kn">from</span> <span class="nn">airflow.hooks.postgres_hook</span> <span class="kn">import</span> <span class="n">PostgresHook</span>
+ <span class="kn">from</span> <span class="nn">airflow.hooks.postgres_hook</span> <span class="k">import</span> <span class="n">PostgresHook</span>
<span class="k">return</span> <span class="n">PostgresHook</span><span class="p">(</span><span class="n">postgres_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
<span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">'hive_cli'</span><span class="p">:</span>
- <span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span>
+ <span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="k">import</span> <span class="n">HiveCliHook</span>
<span class="k">return</span> <span class="n">HiveCliHook</span><span class="p">(</span><span class="n">hive_cli_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
<span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">'presto'</span><span class="p">:</span>
- <span class="kn">from</span> <span class="nn">airflow.hooks.presto_hook</span> <span class="kn">import</span> <span class="n">PrestoHook</span>
+ <span class="kn">from</span> <span class="nn">airflow.hooks.presto_hook</span> <span class="k">import</span> <span class="n">PrestoHook</span>
<span class="k">return</span> <span class="n">PrestoHook</span><span class="p">(</span><span class="n">presto_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
<span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">'hiveserver2'</span><span class="p">:</span>
- <span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveServer2Hook</span>
+ <span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="k">import</span> <span class="n">HiveServer2Hook</span>
<span class="k">return</span> <span class="n">HiveServer2Hook</span><span class="p">(</span><span class="n">hiveserver2_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
<span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">'sqlite'</span><span class="p">:</span>
- <span class="kn">from</span> <span class="nn">airflow.hooks.sqlite_hook</span> <span class="kn">import</span> <span class="n">SqliteHook</span>
+ <span class="kn">from</span> <span class="nn">airflow.hooks.sqlite_hook</span> <span class="k">import</span> <span class="n">SqliteHook</span>
<span class="k">return</span> <span class="n">SqliteHook</span><span class="p">(</span><span class="n">sqlite_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
<span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">'jdbc'</span><span class="p">:</span>
- <span class="kn">from</span> <span class="nn">airflow.hooks.jdbc_hook</span> <span class="kn">import</span> <span class="n">JdbcHook</span>
+ <span class="kn">from</span> <span class="nn">airflow.hooks.jdbc_hook</span> <span class="k">import</span> <span class="n">JdbcHook</span>
<span class="k">return</span> <span class="n">JdbcHook</span><span class="p">(</span><span class="n">jdbc_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
<span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">'mssql'</span><span class="p">:</span>
- <span class="kn">from</span> <span class="nn">airflow.hooks.mssql_hook</span> <span class="kn">import</span> <span class="n">MsSqlHook</span>
+ <span class="kn">from</span> <span class="nn">airflow.hooks.mssql_hook</span> <span class="k">import</span> <span class="n">MsSqlHook</span>
<span class="k">return</span> <span class="n">MsSqlHook</span><span class="p">(</span><span class="n">mssql_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
<span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">'oracle'</span><span class="p">:</span>
- <span class="kn">from</span> <span class="nn">airflow.hooks.oracle_hook</span> <span class="kn">import</span> <span class="n">OracleHook</span>
+ <span class="kn">from</span> <span class="nn">airflow.hooks.oracle_hook</span> <span class="k">import</span> <span class="n">OracleHook</span>
<span class="k">return</span> <span class="n">OracleHook</span><span class="p">(</span><span class="n">oracle_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
<span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">'vertica'</span><span class="p">:</span>
- <span class="kn">from</span> <span class="nn">airflow.contrib.hooks.vertica_hook</span> <span class="kn">import</span> <span class="n">VerticaHook</span>
+ <span class="kn">from</span> <span class="nn">airflow.contrib.hooks.vertica_hook</span> <span class="k">import</span> <span class="n">VerticaHook</span>
<span class="k">return</span> <span class="n">VerticaHook</span><span class="p">(</span><span class="n">vertica_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
<span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">'cloudant'</span><span class="p">:</span>
- <span class="kn">from</span> <span class="nn">airflow.contrib.hooks.cloudant_hook</span> <span class="kn">import</span> <span class="n">CloudantHook</span>
+ <span class="kn">from</span> <span class="nn">airflow.contrib.hooks.cloudant_hook</span> <span class="k">import</span> <span class="n">CloudantHook</span>
<span class="k">return</span> <span class="n">CloudantHook</span><span class="p">(</span><span class="n">cloudant_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+ <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">'jira'</span><s
<TRUNCATED>