You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ka...@apache.org on 2019/12/18 16:51:16 UTC
[airflow-site] branch asf-site updated: Deploy
#fc49cc0aa440e43c3c53649f2e9fe1c770b29896
This is an automated email from the ASF dual-hosted git repository.
kamilbregula pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/airflow-site.git
The following commit(s) were added to refs/heads/asf-site by this push:
new a707ff9 Deploy #fc49cc0aa440e43c3c53649f2e9fe1c770b29896
a707ff9 is described below
commit a707ff96672c93d2b319e9cf43b7f1e8f8da1f2d
Author: Kamil Breguła <ka...@polidea.com>
AuthorDate: Wed Dec 18 17:50:34 2019 +0100
Deploy #fc49cc0aa440e43c3c53649f2e9fe1c770b29896
---
404.html | 12 +-
_gen/docs-docs-index.json | 11 +
_gen/indexes/en/blog-index.json | 2 +-
_gen/indexes/en/blog-posts.json | 2 +-
.../index.html | 1054 +++++++++++++++++++-
blog/airflow-survey/plot1.png | Bin 0 -> 61262 bytes
blog/airflow-survey/plot2.png | Bin 0 -> 64571 bytes
blog/airflow-survey/plot3.png | Bin 0 -> 76203 bytes
.../index.html | 120 ++-
.../index.html | 56 +-
blog/index.html | 90 +-
blog/index.xml | 1026 +++++++++++++++++++
.../index.html | 12 +-
blog/tags/community/index.html | 100 +-
blog/tags/community/index.xml | 1032 ++++++++++++++++++-
blog/tags/development/index.html | 22 +-
blog/tags/{community => survey}/index.html | 56 +-
blog/tags/survey/index.xml | 1006 +++++++++++++++++++
blog/tags/{community => users}/index.html | 56 +-
blog/tags/users/index.xml | 1006 +++++++++++++++++++
categories/index.html | 12 +-
chunk-1.e3a02.js | 1 +
chunk-3.e3a02.js | 1 +
chunk-4.e3a02.js | 19 +
chunk-5.e3a02.js | 51 +
community/index.html | 12 +-
docs.e3a02.js | 1 +
index.html | 42 +-
index.xml | 1026 +++++++++++++++++++
install/index.html | 12 +-
main.e3a02.js | 1 +
meetups/index.html | 12 +-
privacy-notice/index.html | 12 +-
roadmap/index.html | 12 +-
...37660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css | 1 +
search/index.html | 12 +-
sitemap.xml | 34 +-
tags/index.html | 14 +-
tags/index.xml | 2 +-
use-cases/adobe/index.html | 18 +-
use-cases/big-fish-games/index.html | 12 +-
use-cases/dish/index.html | 12 +-
use-cases/experity/index.html | 12 +-
use-cases/index.html | 14 +-
use-cases/onefootball/index.html | 12 +-
45 files changed, 6747 insertions(+), 273 deletions(-)
diff --git a/404.html b/404.html
index 06c54a2..191743f 100644
--- a/404.html
+++ b/404.html
@@ -76,15 +76,15 @@ if (!doNotTrack) {
- <link rel="preload" href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" as="style">
- <link href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" rel="stylesheet" integrity="">
+ <link rel="preload" href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" as="style">
+ <link href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" rel="stylesheet" integrity="">
-<link rel="preload" href="/main.b2992.js" as="script">
-<link rel="preload" href="/chunk-1.b2992.js" as="script">
+<link rel="preload" href="/main.e3a02.js" as="script">
+<link rel="preload" href="/chunk-1.e3a02.js" as="script">
-<link rel="preload" href="/chunk-4.b2992.js" as="script">
+<link rel="preload" href="/chunk-4.e3a02.js" as="script">
@@ -552,6 +552,6 @@ if (!doNotTrack) {
</body>
- <script src="/main.b2992.js"></script>
+ <script src="/main.e3a02.js"></script>
</html>
diff --git a/_gen/docs-docs-index.json b/_gen/docs-docs-index.json
new file mode 100644
index 0000000..c072e9a
--- /dev/null
+++ b/_gen/docs-docs-index.json
@@ -0,0 +1,11 @@
+{
+ "stable": "1.10.6",
+ "versions": [
+ "1.10.1",
+ "1.10.2",
+ "1.10.3",
+ "1.10.4",
+ "1.10.5",
+ "1.10.6"
+ ]
+}
diff --git a/_gen/indexes/en/blog-index.json b/_gen/indexes/en/blog-index.json
index 798e98c..f5d7f33 100644
--- a/_gen/indexes/en/blog-index.json
+++ b/_gen/indexes/en/blog-index.json
@@ -1 +1 @@
-{"version":"2.3.8","fields":["title","description","author","content","tags","url"],"fieldVectors":[["title/It's a \"Breeze\" to develop Apache Airflow",[0,0.953,1,0.13,2,0.234,3,0.234,4,0.413]],["description/It's a \"Breeze\" to develop Apache Airflow",[2,0.279,4,0.334,5,0.771,6,0.369,7,0.771,8,0.369,9,0.369,10,0.369,11,0.105,12,0.369,13,0.771,14,0.369,15,0.771,16,0.083]],["author/It's a \"Breeze\" to develop Apache Airflow",[11,0.095,17,0.334]],["content/It's a \"Breeze\" to develop Ap [...]
\ No newline at end of file
+{"version":"2.3.8","fields":["title","description","author","content","tags","url"],"fieldVectors":[["title/It's a \"Breeze\" to develop Apache Airflow",[0,1.258,1,0.489,2,0.165,3,0.165,4,0.419]],["description/It's a \"Breeze\" to develop Apache Airflow",[2,0.201,4,0.34,5,1.021,6,0.397,7,1.021,8,0.397,9,0.397,10,0.397,11,0.397,12,0.397,13,1.021,14,0.397,15,1.021,16,0.134]],["author/It's a \"Breeze\" to develop Apache Airflow",[11,0.489,17,0.794]],["content/It's a \"Breeze\" to develop Ap [...]
\ No newline at end of file
diff --git a/_gen/indexes/en/blog-posts.json b/_gen/indexes/en/blog-posts.json
index 7325d4a..7ae8603 100644
--- a/_gen/indexes/en/blog-posts.json
+++ b/_gen/indexes/en/blog-posts.json
@@ -1 +1 @@
-[{"content":"## The story behind the Airflow Breeze tool\nInitially, we started contributing to this fantastic open-source project [Apache Airflow] with a team of three which then grew to five. When we kicked it off a year ago, I realized pretty soon where the biggest bottlenecks and areas for improvement in terms of productivity were. Even with the help of our client, who provided us with a “homegrown” development environment it took us literally days to set it up and learn some basics. [...]
\ No newline at end of file
+[{"content":"## The story behind the Airflow Breeze tool\nInitially, we started contributing to this fantastic open-source project [Apache Airflow] with a team of three which then grew to five. When we kicked it off a year ago, I realized pretty soon where the biggest bottlenecks and areas for improvement in terms of productivity were. Even with the help of our client, who provided us with a “homegrown” development environment it took us literally days to set it up and learn some basics. [...]
\ No newline at end of file
diff --git a/blog/its-a-breeze-to-develop-apache-airflow/index.html b/blog/airflow-survey/index.html
similarity index 72%
copy from blog/its-a-breeze-to-develop-apache-airflow/index.html
copy to blog/airflow-survey/index.html
index 6729bfb..0a6c084 100644
--- a/blog/its-a-breeze-to-develop-apache-airflow/index.html
+++ b/blog/airflow-survey/index.html
@@ -29,30 +29,30 @@
<meta name="msapplication-TileImage" content="/favicons/ms-icon-144x144.png">
<meta name="theme-color" content="#ffffff">
-<title>It's a "Breeze" to develop Apache Airflow | Apache Airflow</title><meta property="og:title" content="It's a "Breeze" to develop Apache Airflow" />
-<meta property="og:description" content="A Principal Software Engineer's journey to developer productivity. Learn how Jarek and his team speeded up and simplified Airflow development for the community." />
+<title>Airflow Survey 2019 | Apache Airflow</title><meta property="og:title" content="Airflow Survey 2019" />
+<meta property="og:description" content="Receiving and adjusting to our users’ feedback is a must. Let’s see who Airflow users are, how they play with it, and what they miss." />
<meta property="og:type" content="article" />
-<meta property="og:url" content="/blog/its-a-breeze-to-develop-apache-airflow/" />
+<meta property="og:url" content="/blog/airflow-survey/" />
<meta property="og:image" content="/images/feature-image.png" />
-<meta property="article:published_time" content="2019-11-22T00:00:00+00:00" />
-<meta property="article:modified_time" content="2019-11-26T12:05:52+01:00" /><meta property="og:site_name" content="Apache Airflow" />
-<meta itemprop="name" content="It's a "Breeze" to develop Apache Airflow">
-<meta itemprop="description" content="A Principal Software Engineer's journey to developer productivity. Learn how Jarek and his team speeded up and simplified Airflow development for the community.">
+<meta property="article:published_time" content="2019-12-11T00:00:00+00:00" />
+<meta property="article:modified_time" content="2019-12-18T17:24:41+01:00" /><meta property="og:site_name" content="Apache Airflow" />
+<meta itemprop="name" content="Airflow Survey 2019">
+<meta itemprop="description" content="Receiving and adjusting to our users’ feedback is a must. Let’s see who Airflow users are, how they play with it, and what they miss.">
-<meta itemprop="datePublished" content="2019-11-22T00:00:00+00:00" />
-<meta itemprop="dateModified" content="2019-11-26T12:05:52+01:00" />
-<meta itemprop="wordCount" content="127">
+<meta itemprop="datePublished" content="2019-12-11T00:00:00+00:00" />
+<meta itemprop="dateModified" content="2019-12-18T17:24:41+01:00" />
+<meta itemprop="wordCount" content="1775">
-<meta itemprop="keywords" content="Development," />
+<meta itemprop="keywords" content="community,survey,users," />
<meta name="twitter:card" content="summary_large_image"/>
<meta name="twitter:image" content="/images/feature-image.png"/>
-<meta name="twitter:title" content="It's a "Breeze" to develop Apache Airflow"/>
-<meta name="twitter:description" content="A Principal Software Engineer's journey to developer productivity. Learn how Jarek and his team speeded up and simplified Airflow development for the community."/>
+<meta name="twitter:title" content="Airflow Survey 2019"/>
+<meta name="twitter:description" content="Receiving and adjusting to our users’ feedback is a must. Let’s see who Airflow users are, how they play with it, and what they miss."/>
<script type="application/javascript">
@@ -80,20 +80,20 @@ if (!doNotTrack) {
crossorigin="anonymous"></script>
-<meta name="description" content="A Principal Software Engineer's journey to developer productivity. Learn how Jarek and his team speeded up and simplified Airflow development for the community." />
+<meta name="description" content="Receiving and adjusting to our users’ feedback is a must. Let’s see who Airflow users are, how they play with it, and what they miss." />
- <link rel="preload" href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" as="style">
- <link href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" rel="stylesheet" integrity="">
+ <link rel="preload" href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" as="style">
+ <link href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" rel="stylesheet" integrity="">
-<link rel="preload" href="/main.b2992.js" as="script">
-<link rel="preload" href="/chunk-1.b2992.js" as="script">
+<link rel="preload" href="/main.e3a02.js" as="script">
+<link rel="preload" href="/chunk-1.e3a02.js" as="script">
-<link rel="preload" href="/chunk-4.b2992.js" as="script">
+<link rel="preload" href="/chunk-4.e3a02.js" as="script">
@@ -379,19 +379,27 @@ if (!doNotTrack) {
<div class="tags-container">
- <a class="tag" href="/blog/tags/development/">Development</a>
+ <a class="tag" href="/blog/tags/community/">Community</a>
+
+
+
+ <a class="tag" href="/blog/tags/survey/">Survey</a>
+
+
+
+ <a class="tag" href="/blog/tags/users/">Users</a>
</div>
- <span class="bodytext__medium--brownish-grey">Fri, Nov 22, 2019</span>
+ <span class="bodytext__medium--brownish-grey">Wed, Dec 11, 2019</span>
</div>
- <p class="blogpost-content__metadata--title">It's a "Breeze" to develop Apache Airflow</p>
+ <p class="blogpost-content__metadata--title">Airflow Survey 2019</p>
<div class="blogpost-content__metadata--author">
<span class="blogpost-content__metadata--author">
- Jarek Potiuk
+ Tomek Urbaszek
</span>
- <a href="https://twitter.com/higrys/" class="blogpost-content__metadata--social-media-icon">
+ <a href="https://twitter.com/Nuclearriot/" class="blogpost-content__metadata--social-media-icon">
<svg xmlns="http://www.w3.org/2000/svg" width="22" height="21" viewBox="0 0 22 21">
<g id="Group_1746" data-name="Group 1746" transform="translate(.076 -.055)">
<ellipse id="Ellipse_19" cx="11" cy="10.5" fill="#51504f" data-name="Ellipse 19" rx="11" ry="10.5"
@@ -407,7 +415,7 @@ if (!doNotTrack) {
</a>
- <a href="https://github.com/potiuk/" class="blogpost-content__metadata--social-media-icon">
+ <a href="https://github.com/nuclearpinguin/" class="blogpost-content__metadata--social-media-icon">
<svg xmlns="http://www.w3.org/2000/svg" width="21.737" height="21.2" viewBox="0 0 21.737 21.2">
<path id="Path_1378" d="M33.971 1181.31a10.87 10.87 0 0 0-3.435 21.182c.543.1.742-.236.742-.524 0-.258-.009-.941-.015-1.848-3.023.657-3.661-1.457-3.661-1.457a2.876 2.876 0 0 0-1.207-1.59c-.987-.674.075-.661.075-.661a2.283 2.283 0 0 1 1.665 1.12 2.314 2.314 0 0 0 3.163.9 2.322 2.322 0 0 1 .69-1.453c-2.413-.274-4.951-1.207-4.951-5.371a4.2 4.2 0 0 1 1.119-2.917 3.908 3.908 0 0 1 .107-2.876s.913-.292 2.989 1.114a10.3 10.3 0 0 1 5.442 0c2.075-1.406 2.986-1.114 2.986-1.114a3.9 3.9 0 0 1 .1 [...]
</svg>
@@ -415,7 +423,7 @@ if (!doNotTrack) {
</a>
- <a href="https://linkedin.com/in/jarekpotiuk/" class="blogpost-content__metadata--social-media-icon">
+ <a href="https://linkedin.com/in/tomaszurbaszek/" class="blogpost-content__metadata--social-media-icon">
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 20 21">
<g id="Group_1745" data-name="Group 1745" transform="translate(.155 -.055)">
<ellipse id="Ellipse_20" cx="10" cy="10.5" fill="#51504f" data-name="Ellipse 20" rx="10" ry="10.5" transform="translate(-.155 .055)"/>
@@ -429,18 +437,984 @@ if (!doNotTrack) {
</a>
</div>
- <p class="blogpost-content__metadata--description">A Principal Software Engineer's journey to developer productivity. Learn how Jarek and his team speeded up and simplified Airflow development for the community.</p>
+ <p class="blogpost-content__metadata--description">Receiving and adjusting to our users’ feedback is a must. Let’s see who Airflow users are, how they play with it, and what they miss.</p>
</div>
<div class="markdown-content">
-<h2 id="the-story-behind-the-airflow-breeze-tool">The story behind the Airflow Breeze tool</h2>
-
-<p>Initially, we started contributing to this fantastic open-source project [Apache Airflow] with a team of three which then grew to five. When we kicked it off a year ago, I realized pretty soon where the biggest bottlenecks and areas for improvement in terms of productivity were. Even with the help of our client, who provided us with a “homegrown” development environment it took us literally days to set it up and learn some basics.</p>
-
-<p>That is how the journey to increased productivity in Apache Airflow began. The result? The Airflow Breeze open-source tool. Jarek Potiuk, an Airflow Committer, will tell you all about it.</p>
-
-<p>You can learn <a href="https://www.polidea.com/blog/its-a-breeze-to-develop-apache-airflow/?utm_source=ApacheAirflowBlog&utm_medium=Npaid&utm_campaign=Blog&utm_term=Article&utm_content=AAB_NOP_BLG_ART_AB_001" target="_blank">how and why it’s a “Breeze” to Develop Apache Airflow on Polidea blog</a>.</p>
+<h1 id="apache-airflow-survey-2019">Apache Airflow Survey 2019</h1>
+
+<p>Apache Airflow is <a href="https://www.astronomer.io/blog/why-airflow/" target="_blank">growing faster than ever</a>.
+Thus, receiving and adjusting to our users’ feedback is a must. We created
+<a href="https://forms.gle/XAzR1pQBZiftvPQM7" target="_blank">survey</a> and we got <strong>308</strong> responses.
+Let’s see who Airflow users are, how they play with it, and what they miss.</p>
+
+<h1 id="overview-of-the-user">Overview of the user</h1>
+
+<p><strong>What best describes your current occupation?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Data Engineer</td>
+<td>194</td>
+<td>62.99%</td>
+</tr>
+
+<tr>
+<td>Developer</td>
+<td>34</td>
+<td>11.04%</td>
+</tr>
+
+<tr>
+<td>Architect</td>
+<td>23</td>
+<td>7.47%</td>
+</tr>
+
+<tr>
+<td>Data Scientist</td>
+<td>19</td>
+<td>6.17%</td>
+</tr>
+
+<tr>
+<td>Data Analyst</td>
+<td>13</td>
+<td>4.22%</td>
+</tr>
+
+<tr>
+<td>DevOps</td>
+<td>13</td>
+<td>4.22%</td>
+</tr>
+
+<tr>
+<td>IT Administrator</td>
+<td>2</td>
+<td>0.65%</td>
+</tr>
+
+<tr>
+<td>Machine Learning Engineer</td>
+<td>2</td>
+<td>0.65%</td>
+</tr>
+
+<tr>
+<td>Manager</td>
+<td>2</td>
+<td>0.65%</td>
+</tr>
+
+<tr>
+<td>Operations</td>
+<td>2</td>
+<td>0.65%</td>
+</tr>
+
+<tr>
+<td>Chief Data Officer</td>
+<td>1</td>
+<td>0.32%</td>
+</tr>
+
+<tr>
+<td>Engineering Manager</td>
+<td>1</td>
+<td>0.32%</td>
+</tr>
+
+<tr>
+<td>Intern</td>
+<td>1</td>
+<td>0.32%</td>
+</tr>
+
+<tr>
+<td>Product owner</td>
+<td>1</td>
+<td>0.32%</td>
+</tr>
+
+<tr>
+<td>Quant</td>
+<td>1</td>
+<td>0.32%</td>
+</tr>
+</tbody>
+</table>
+
+<p><strong>In your day to day job, what do you use Airflow for?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Data processing (ETL)</td>
+<td>298</td>
+<td>96.75%</td>
+</tr>
+
+<tr>
+<td>Artificial Intelligence and Machine Learning Pipelines</td>
+<td>90</td>
+<td>29.22%</td>
+</tr>
+
+<tr>
+<td>Automating DevOps operations</td>
+<td>64</td>
+<td>20.78%</td>
+</tr>
+</tbody>
+</table>
+
+<p>According to the survey, most of the Airflow users are the “data” people. Moreover,
+28.57% uses Airflow to both ETL and ML pipelines meaning that those two fields
+are somehow connected. Only five respondents use Airflow for DevOps operations only,
+That means that other 59 people who use Airflow for DevOps stuff use it also for
+ETL / ML purposes.</p>
+
+<p><strong>How many active DAGs do you have in your largest Airflow instance?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>0-20</td>
+<td>115</td>
+<td>37.34%</td>
+</tr>
+
+<tr>
+<td>21-40</td>
+<td>65</td>
+<td>21.10%</td>
+</tr>
+
+<tr>
+<td>41-60</td>
+<td>44</td>
+<td>14.29%</td>
+</tr>
+
+<tr>
+<td>61-100</td>
+<td>28</td>
+<td>9.09%</td>
+</tr>
+
+<tr>
+<td>101-200</td>
+<td>28</td>
+<td>9.09%</td>
+</tr>
+
+<tr>
+<td>201-300</td>
+<td>7</td>
+<td>2.27%</td>
+</tr>
+
+<tr>
+<td>301-999</td>
+<td>8</td>
+<td>2.60%</td>
+</tr>
+
+<tr>
+<td>1000+</td>
+<td>13</td>
+<td>4.22%</td>
+</tr>
+</tbody>
+</table>
+
+<p>The majority of users do not exceed 100 active DAGs per Airflow instance. However,
+as we can see there are users who exceed thousands of DAGs with a maximum number 5000.</p>
+
+<p><strong>What is the maximum number of tasks that you have used in one DAG?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>0-10</td>
+<td>61</td>
+<td>19.81%</td>
+</tr>
+
+<tr>
+<td>11-20</td>
+<td>60</td>
+<td>19.48%</td>
+</tr>
+
+<tr>
+<td>21-30</td>
+<td>31</td>
+<td>10.06%</td>
+</tr>
+
+<tr>
+<td>31-40</td>
+<td>21</td>
+<td>6.82%</td>
+</tr>
+
+<tr>
+<td>41-50</td>
+<td>26</td>
+<td>8.44%</td>
+</tr>
+
+<tr>
+<td>51-100</td>
+<td>36</td>
+<td>11.69%</td>
+</tr>
+
+<tr>
+<td>101-200</td>
+<td>28</td>
+<td>9.09%</td>
+</tr>
+
+<tr>
+<td>201-500</td>
+<td>21</td>
+<td>6.82%</td>
+</tr>
+
+<tr>
+<td>501+</td>
+<td>24</td>
+<td>11.54%</td>
+</tr>
+</tbody>
+</table>
+
+<p>The given maximum number of tasks in a single DAG was 10 000 (!). The number of tasks
+depends on the purposes of a DAG, so it’s rather hard to say if users have “simple”
+or “complicated” workflows.</p>
+
+<p><strong>When onboarding new members to Airflow, what is the biggest problem?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>No guide on best practises on developing DAGs</td>
+<td>160</td>
+<td>51.95%</td>
+</tr>
+
+<tr>
+<td>Small number of tutorials on different aspects of using Airflow</td>
+<td>57</td>
+<td>18.51%</td>
+</tr>
+
+<tr>
+<td>Documentation is not clear enough</td>
+<td>42</td>
+<td>13.64%</td>
+</tr>
+
+<tr>
+<td>Small number of blogs regarding Airflow</td>
+<td>6</td>
+<td>1.95%</td>
+</tr>
+
+<tr>
+<td>Other</td>
+<td>43</td>
+<td>13.96%</td>
+</tr>
+</tbody>
+</table>
+
+<p>This is an important result. Using Airflow is all about writing and scheduling DAGs.
+No guide or any other complete resource on best practices for developing Dags is a big
+problem. Diving deep in the “other” answers, we can find that:</p>
+
+<ul>
+<li>Airflow’s “magic” (scheduler, executors, schedule times) is hard to understand</li>
+<li>DAG testing is not easy to do and to explain</li>
+<li>Airflow UI needs some love.</li>
+</ul>
+
+<p><strong>How likely are you to recommend Apache Airflow?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Very Likely</td>
+<td>140</td>
+<td>45.45%</td>
+</tr>
+
+<tr>
+<td>Likely</td>
+<td>124</td>
+<td>40.26%</td>
+</tr>
+
+<tr>
+<td>Neutral</td>
+<td>33</td>
+<td>10.71%</td>
+</tr>
+
+<tr>
+<td>Unlikely</td>
+<td>8</td>
+<td>2.60%</td>
+</tr>
+
+<tr>
+<td>Very unlikely</td>
+<td>3</td>
+<td>0.97%</td>
+</tr>
+</tbody>
+</table>
+
+<p>This means that more than 85% of people who use Airflow like it. It seems Airflow does
+its job nicely. However, we have to remember that this survey is likely biased - it’s
+more likely that you respond to the survey if you like the tool you use. Should we
+focus then on those 11 people who did not like Airflow? It’s a good question.</p>
+
+<h2 id="airflow-usage">Airflow usage</h2>
+
+<p><strong>Which interface(s) of Airflow do you use as part of your current role?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Original Airflow Graphical User Interface</td>
+<td>297</td>
+<td>96.43%</td>
+</tr>
+
+<tr>
+<td>CLI</td>
+<td>126</td>
+<td>40.91%</td>
+</tr>
+
+<tr>
+<td>Original Airflow Graphical User Interface, CLI</td>
+<td>117</td>
+<td>37.99%</td>
+</tr>
+
+<tr>
+<td>API</td>
+<td>60</td>
+<td>19.48%</td>
+</tr>
+
+<tr>
+<td>Original Airflow Graphical User Interface, CLI, API</td>
+<td>32</td>
+<td>10.39%</td>
+</tr>
+
+<tr>
+<td>Custom (own created) Airflow Graphical User Interface</td>
+<td>25</td>
+<td>8.12%</td>
+</tr>
+</tbody>
+</table>
+
+<p>It’s visible that usage of CLI goes in pair with using Airflow web UI. Our
+survey included some UX related questions to allow us to understand how users
+use Airflow webserver.</p>
+
+<p><strong>What do you use the Graphical User Interface for?</strong></p>
+
+<p><img src="plot1.png" alt="" /></p>
+
+<p><strong>What do you use CLI for?</strong></p>
+
+<p><img src="plot2.png" alt="" /></p>
+
+<p><strong>In Airflow, which UI view(s) are important for you?</strong></p>
+
+<p><img src="plot3.png" alt="" /></p>
+
+<p>Here we see that the majority uses Web UI mostly for monitoring purposes:</p>
+
+<ul>
+<li>Monitoring DAGs</li>
+<li>Accessing logs</li>
+</ul>
+
+<p>An interesting result is that many people seem not to use backfilling as
+there’s no other way than to do it by CLI.</p>
+
+<p><strong>What executor type do you use?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Celery</td>
+<td>138</td>
+<td>44.81%</td>
+</tr>
+
+<tr>
+<td>Local</td>
+<td>85</td>
+<td>27.60%</td>
+</tr>
+
+<tr>
+<td>Kubernetes</td>
+<td>52</td>
+<td>16.88%</td>
+</tr>
+
+<tr>
+<td>Sequential</td>
+<td>22</td>
+<td>7.14%</td>
+</tr>
+
+<tr>
+<td>Other</td>
+<td>11</td>
+<td>3.57</td>
+</tr>
+</tbody>
+</table>
+
+<p>The other option mostly consisted of information that someone uses a few types or is
+migrating from one executor to another. What can be observed is an increase in usage
+of Local and Kubernetes executors when compared to results from an earlier <a href="https://ash.berlintaylor.com/writings/2019/02/airflow-user-survey-2019/" target="_blank">survey done
+by Ash</a>.</p>
+
+<p><strong>Do you use Kubernetes-based deployments for Airflow?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>No - we do not plan to use Kubernetes near term</td>
+<td>88</td>
+<td>28.57%</td>
+</tr>
+
+<tr>
+<td>Yes - setup on our own via Helm Chart or similar</td>
+<td>65</td>
+<td>21.10%</td>
+</tr>
+
+<tr>
+<td>Not yet - but we use Kubernetes in our organization and we could move</td>
+<td>61</td>
+<td>19.81%</td>
+</tr>
+
+<tr>
+<td>Yes - via managed service in the cloud (Composer / Astronomer etc.)</td>
+<td>45</td>
+<td>14.61%</td>
+</tr>
+
+<tr>
+<td>Not yet - but we plan to deploy Kubernetes in our organization soon</td>
+<td>42</td>
+<td>13.64%</td>
+</tr>
+
+<tr>
+<td>Other</td>
+<td>7</td>
+<td>2.27%</td>
+</tr>
+</tbody>
+</table>
+
+<p>The most interesting thing is that there’s nearly 30% of users who do not use Kubernetes,
+and they are not going to move. This means we should keep other deployment options in
+mind when working on Airflow 2.0. On the other hand, almost 70% of the users already
+use Kubernetes, or it’s a viable option for them.</p>
+
+<p><strong>Do you combine multiple DAGs?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>No, I don’t combine multiple DAGs</td>
+<td>127</td>
+<td>41.23%</td>
+</tr>
+
+<tr>
+<td>Yes, through SubDAG</td>
+<td>73</td>
+<td>23.70%</td>
+</tr>
+
+<tr>
+<td>Yes, by triggering another DAG</td>
+<td>72</td>
+<td>23.38%</td>
+</tr>
+
+<tr>
+<td>Other</td>
+<td>36</td>
+<td>11.69%</td>
+</tr>
+</tbody>
+</table>
+
+<p>In the other category, 9 people explicitly mentioned using <code>ExternalTaskSensor</code>,
+and I think it could be treated as running subDAGs by triggering other DAGs.</p>
+
+<p><strong>Do you use Airflow Plugins? If yes, what do you use it for?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Adding new operators/sensors and hooks</td>
+<td>187</td>
+<td>60.71%</td>
+</tr>
+
+<tr>
+<td>I don’t use Airflow plugins</td>
+<td>109</td>
+<td>35.39%</td>
+</tr>
+
+<tr>
+<td>Adding AppBuilder views & menu items</td>
+<td>31</td>
+<td>10.06%</td>
+</tr>
+
+<tr>
+<td>Adding new executor</td>
+<td>18</td>
+<td>5.84%</td>
+</tr>
+
+<tr>
+<td>Adding OperatorExtraLinks</td>
+<td>7</td>
+<td>2.27%</td>
+</tr>
+</tbody>
+</table>
+
+<p>The high percentage - 60% for “Adding new operators/sensors and hooks” is quite a
+surprising result for some of us - especially that you do not actually need to use the
+plugin mechanism to add any of those. Those are standard python objects, and you can
+simply drop your hooks/operators/sensors code to <code>PYTHONPATH</code> environment variable and
+they will work. It seems that this may be a result of a lack of best practices guide.</p>
+
+<p>Plugins are more useful for adding views and menu items - yet only 10%.
+OperatorExtraLinks are even more useful (though relatively new) feature, so it’s not
+entirely surprising they are hardly used.</p>
+
+<p>It was also kind of surprising that someone at all uses plugins to use their own
+executors. We considered removing that option recently - but now we have to rethink
+our approach.</p>
+
+<p><strong>What metrics do you use to monitor Airflow?</strong></p>
+
+<p>There were a lot of different responses. Some use Prometheus and other services,
+others do not use any monitoring. One of the interesting responses linked to this
+solution for <a href="https://github.com/mastak/airflow_operators_metrics" target="_blank">airflow_operators_metrics</a>.</p>
+
+<h2 id="external-services">External services</h2>
+
+<p><strong>What external services do you use in your Airflow DAGs?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Amazon Web Services</td>
+<td>160</td>
+<td>51.95%</td>
+</tr>
+
+<tr>
+<td>Internal company systems</td>
+<td>150</td>
+<td>48.7%</td>
+</tr>
+
+<tr>
+<td>Hadoop / Spark / Flink / Other Apache software</td>
+<td>119</td>
+<td>38.64%</td>
+</tr>
+
+<tr>
+<td>Google Cloud Platform / Google APIs</td>
+<td>112</td>
+<td>36.36%</td>
+</tr>
+
+<tr>
+<td>Microsoft Azure</td>
+<td>28</td>
+<td>9.09%</td>
+</tr>
+
+<tr>
+<td>I do not use external services in my Airflow DAGs</td>
+<td>18</td>
+<td>5.84%</td>
+</tr>
+</tbody>
+</table>
+
+<p>It’s not surprising that Amazon Web Services is leading the way as they are considered the most mature
+cloud provider. Internal system and other Apache products on the next two positions are
+quite understandable if we take into account that the majority uses Airflow for ETL processes.</p>
+
+<p><strong>What external services do you use in your Airflow DAGs? (Mixed providers)</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Google Cloud Platform / Google APIs, Amazon Web Services</td>
+<td>44</td>
+<td>14.29%</td>
+</tr>
+
+<tr>
+<td>Amazon Web Services, Microsoft Azure</td>
+<td>5</td>
+<td>1.62%</td>
+</tr>
+
+<tr>
+<td>Google Cloud Platform / Google APIs, Microsoft Azure</td>
+<td>4</td>
+<td>1.3%</td>
+</tr>
+</tbody>
+</table>
+
+<p>This result is not surprising because companies usually prefer to stick with one cloud
+provider.</p>
+
+<p><strong>How do you integrate with external services?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Using Bash / Python operator</td>
+<td>220</td>
+<td>71.43%</td>
+</tr>
+
+<tr>
+<td>Using existing, dedicated operators / hooks</td>
+<td>217</td>
+<td>70.45%</td>
+</tr>
+
+<tr>
+<td>Using own, custom operators / hooks</td>
+<td>216</td>
+<td>70.13%</td>
+</tr>
+</tbody>
+</table>
+
+<p>We had some anecdotal evidence that people use more Python/Bash operators than the
+dedicated ones - but it looks like all ways of using Airflow to connect to external
+services are equally popular.</p>
+
+<h2 id="what-can-be-improved">What can be improved</h2>
+
+<p><strong>In your opinion, what could be improved in Airflow?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Scheduler performance</td>
+<td>189</td>
+<td>61.36%</td>
+</tr>
+
+<tr>
+<td>Web UI</td>
+<td>180</td>
+<td>58.44%</td>
+</tr>
+
+<tr>
+<td>Logging, monitoring and alerting</td>
+<td>145</td>
+<td>47.08%</td>
+</tr>
+
+<tr>
+<td>Examples, how-to, onboarding documentation</td>
+<td>143</td>
+<td>46.43%</td>
+</tr>
+
+<tr>
+<td>Technical documentation</td>
+<td>137</td>
+<td>44.48%</td>
+</tr>
+
+<tr>
+<td>Reliability</td>
+<td>112</td>
+<td>36.36%</td>
+</tr>
+
+<tr>
+<td>REST API</td>
+<td>96</td>
+<td>31.17%</td>
+</tr>
+
+<tr>
+<td>Authentication and authorization</td>
+<td>89</td>
+<td>28.9%</td>
+</tr>
+
+<tr>
+<td>External integration e.g. AWS, GCP, Apache product</td>
+<td>49</td>
+<td>15.91%</td>
+</tr>
+
+<tr>
+<td>CLI</td>
+<td>41</td>
+<td>13.31%</td>
+</tr>
+
+<tr>
+<td>I don’t know</td>
+<td>5</td>
+<td>1.62%</td>
+</tr>
+</tbody>
+</table>
+
+<p>The results are rather quite self-explaining. Improved performance of Airflow, better
+UI, and more telemetry are desirable. But this should go in pair with improved
+documentation and resources about using the Airflow, especially when we
+take into account the problem of onboarding new users.</p>
+
+<p>Another interesting point from that question is that only 16% think that operators
+should be extended and improved. This suggests that we should focus on improving
+Airflow core instead of adding more and more integrations.</p>
+
+<p><strong>What would be the most interesting feature for you?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Production-ready Airflow docker image</td>
+<td>175</td>
+<td>56.82%</td>
+</tr>
+
+<tr>
+<td>Declarative way of writing DAGs / automated DAGs generation</td>
+<td>155</td>
+<td>50.32%</td>
+</tr>
+
+<tr>
+<td>Horizontal Autoscaling</td>
+<td>122</td>
+<td>39.61%</td>
+</tr>
+
+<tr>
+<td>Asynchronous Operators</td>
+<td>97</td>
+<td>31.49%</td>
+</tr>
+
+<tr>
+<td>Stateless web server</td>
+<td>81</td>
+<td>26.3%</td>
+</tr>
+
+<tr>
+<td>Knative Executor</td>
+<td>48</td>
+<td>15.58%</td>
+</tr>
+
+<tr>
+<td>I already have all I need</td>
+<td>13</td>
+<td>4.22%</td>
+</tr>
+</tbody>
+</table>
+
+<p>Production Docker image wins, and it’s not a surprise. We all know that deploying
+Airflow is not a plug and play process, and that’s why the official image is being
+worked on by Jarek Potiuk. An unexpected result is that half of the users would like to
+have a declarative way of creating DAGs. That seems to be something that is “against Airflow”
+as we always emphasize the possibility of writing workflows in pure python. Stories
+about DAG generators are not new and confirm that there’s a need for a way to
+declare DAGs.</p>
+
+<h2 id="data">Data</h2>
+
+<p>If you think I missed something and you want to look for insights on your own the data is available
+for you here:</p>
+
+<ul>
+<li>Original data: <a href="https://storage.googleapis.com/airflow-survey/survey.csv" target="_blank">https://storage.googleapis.com/airflow-survey/survey.csv</a></li>
+<li>Processed: <a href="https://storage.googleapis.com/airflow-survey/airflow_survey_processed.csv" target="_blank">https://storage.googleapis.com/airflow-survey/airflow_survey_processed.csv</a></li>
+</ul>
+
+<p>The processed data includes multi-choice options one-hot encoded. If you find any interesting
+insight, please update the article (<a href="https://github.com/apache/airflow-site/blob/aip-11/CONTRIBUTE.md" target="_blank">make PR</a>
+to Airflow site).</p>
</div>
@@ -498,16 +1472,16 @@ if (!doNotTrack) {
<div class="pager">
- <a >
+ <a href="/blog/announcing-new-website/">
-<button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" disabled>Previous</button>
+<button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" >Previous</button>
</a>
- <a href="/blog/apache-con-europe-2019-thoughts-and-insights-by-airflow-commiters/">
+ <a >
-<button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" >Next</button>
+<button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" disabled>Next</button>
</a>
</div>
@@ -528,7 +1502,7 @@ if (!doNotTrack) {
<div class="base-layout--button">
- <a href=https://github.com/apache/airflow-site/edit/aip-11/landing-pages/site/content/en/blog/Its-a-breeze-to-develop-apache-airflow.md>
+ <a href=https://github.com/apache/airflow-site/edit/aip-11/landing-pages/site/content/en/blog/airflow-survey/index.md>
@@ -706,6 +1680,6 @@ if (!doNotTrack) {
</body>
- <script src="/main.b2992.js"></script>
+ <script src="/main.e3a02.js"></script>
</html>
diff --git a/blog/airflow-survey/plot1.png b/blog/airflow-survey/plot1.png
new file mode 100644
index 0000000..57be032
Binary files /dev/null and b/blog/airflow-survey/plot1.png differ
diff --git a/blog/airflow-survey/plot2.png b/blog/airflow-survey/plot2.png
new file mode 100644
index 0000000..f72112b
Binary files /dev/null and b/blog/airflow-survey/plot2.png differ
diff --git a/blog/airflow-survey/plot3.png b/blog/airflow-survey/plot3.png
new file mode 100644
index 0000000..63a20a3
Binary files /dev/null and b/blog/airflow-survey/plot3.png differ
diff --git a/blog/apache-con-europe-2019-thoughts-and-insights-by-airflow-commiters/index.html b/blog/announcing-new-website/index.html
similarity index 90%
copy from blog/apache-con-europe-2019-thoughts-and-insights-by-airflow-commiters/index.html
copy to blog/announcing-new-website/index.html
index 93d8820..1c4825d 100644
--- a/blog/apache-con-europe-2019-thoughts-and-insights-by-airflow-commiters/index.html
+++ b/blog/announcing-new-website/index.html
@@ -29,21 +29,21 @@
<meta name="msapplication-TileImage" content="/favicons/ms-icon-144x144.png">
<meta name="theme-color" content="#ffffff">
-<title>ApacheCon Europe 2019 — Thoughts and Insights by Airflow Commiters | Apache Airflow</title><meta property="og:title" content="ApacheCon Europe 2019 — Thoughts and Insights by Airflow Commiters" />
-<meta property="og:description" content="Here come some thoughts by Airflow committers and contributors from the ApacheCon Europe 2019. Get to know the ASF community!" />
+<title>New Airflow website | Apache Airflow</title><meta property="og:title" content="New Airflow website" />
+<meta property="og:description" content="We are thrilled about our new website!" />
<meta property="og:type" content="article" />
-<meta property="og:url" content="/blog/apache-con-europe-2019-thoughts-and-insights-by-airflow-commiters/" />
+<meta property="og:url" content="/blog/announcing-new-website/" />
<meta property="og:image" content="/images/feature-image.png" />
-<meta property="article:published_time" content="2019-11-22T00:00:00+00:00" />
-<meta property="article:modified_time" content="2019-11-26T12:05:52+01:00" /><meta property="og:site_name" content="Apache Airflow" />
-<meta itemprop="name" content="ApacheCon Europe 2019 — Thoughts and Insights by Airflow Commiters">
-<meta itemprop="description" content="Here come some thoughts by Airflow committers and contributors from the ApacheCon Europe 2019. Get to know the ASF community!">
+<meta property="article:published_time" content="2019-12-11T00:00:00+00:00" />
+<meta property="article:modified_time" content="2019-12-18T08:19:54-08:00" /><meta property="og:site_name" content="Apache Airflow" />
+<meta itemprop="name" content="New Airflow website">
+<meta itemprop="description" content="We are thrilled about our new website!">
-<meta itemprop="datePublished" content="2019-11-22T00:00:00+00:00" />
-<meta itemprop="dateModified" content="2019-11-26T12:05:52+01:00" />
-<meta itemprop="wordCount" content="138">
+<meta itemprop="datePublished" content="2019-12-11T00:00:00+00:00" />
+<meta itemprop="dateModified" content="2019-12-18T08:19:54-08:00" />
+<meta itemprop="wordCount" content="282">
@@ -51,8 +51,8 @@
<meta name="twitter:card" content="summary_large_image"/>
<meta name="twitter:image" content="/images/feature-image.png"/>
-<meta name="twitter:title" content="ApacheCon Europe 2019 — Thoughts and Insights by Airflow Commiters"/>
-<meta name="twitter:description" content="Here come some thoughts by Airflow committers and contributors from the ApacheCon Europe 2019. Get to know the ASF community!"/>
+<meta name="twitter:title" content="New Airflow website"/>
+<meta name="twitter:description" content="We are thrilled about our new website!"/>
<script type="application/javascript">
@@ -80,20 +80,20 @@ if (!doNotTrack) {
crossorigin="anonymous"></script>
-<meta name="description" content="Here come some thoughts by Airflow committers and contributors from the ApacheCon Europe 2019. Get to know the ASF community!" />
+<meta name="description" content="We are thrilled about our new website!" />
- <link rel="preload" href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" as="style">
- <link href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" rel="stylesheet" integrity="">
+ <link rel="preload" href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" as="style">
+ <link href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" rel="stylesheet" integrity="">
-<link rel="preload" href="/main.b2992.js" as="script">
-<link rel="preload" href="/chunk-1.b2992.js" as="script">
+<link rel="preload" href="/main.e3a02.js" as="script">
+<link rel="preload" href="/chunk-1.e3a02.js" as="script">
-<link rel="preload" href="/chunk-4.b2992.js" as="script">
+<link rel="preload" href="/chunk-4.e3a02.js" as="script">
@@ -383,25 +383,45 @@ if (!doNotTrack) {
</div>
- <span class="bodytext__medium--brownish-grey">Fri, Nov 22, 2019</span>
+ <span class="bodytext__medium--brownish-grey">Wed, Dec 11, 2019</span>
</div>
- <p class="blogpost-content__metadata--title">ApacheCon Europe 2019 — Thoughts and Insights by Airflow Commiters</p>
+ <p class="blogpost-content__metadata--title">New Airflow website</p>
<div class="blogpost-content__metadata--author">
<span class="blogpost-content__metadata--author">
- Polidea
+ Aizhamal Nurmamat kyzy
</span>
</div>
- <p class="blogpost-content__metadata--description">Here come some thoughts by Airflow committers and contributors from the ApacheCon Europe 2019. Get to know the ASF community!</p>
+ <p class="blogpost-content__metadata--description">We are thrilled about our new website!</p>
</div>
<div class="markdown-content">
- <p>Is it possible to create an organization that delivers tens of projects used by millions, nearly no one is paid for doing their job, and still, it has been fruitfully carrying on for more than 20 years? Apache Software Foundation proves it is possible. For the last two decades, ASF has been crafting a model called the Apache Way—a way of organizing and leading tech open source projects. Due to this approach, which is strongly based on the “community over code” motto, we can enjoy [...]
+ <p>The brand <a href="https://airflow.apache.org/" target="_blank">new Airflow website</a> has arrived! Those who have been following the process know that the journey to update <a href="https://airflow.readthedocs.io/en/1.10.6/" target="_blank">the old Airflow website</a> started at the beginning of the year.
+Thanks to sponsorship from the Cloud Composer team at Google that allowed us to
+collaborate with <a href="https://www.polidea.com/" target="_blank">Polidea</a> and with their design studio <a href="https://utilodesign.com/" target="_blank">Utilo</a>, and deliver an awesome website.</p>
-<p>After this year’s ApacheCon, Polidea’s engineers talked with Committers of Apache projects, such as—Aizhamal Nurmamat kyzy, Felix Uellendall, and Fokko Driesprong—about insights to what makes the ASF such an amazing organization.</p>
+<p>Documentation of open source projects is key to engaging new contributors in the maintenance,
+development, and adoption of software. We want the Apache Airflow community to have
+the best possible experience to contribute and use the project. We also took this opportunity to make the project
+more accessible, and in doing so, increase its reach.</p>
-<p>You can read the <a href="https://www.polidea.com/blog/apachecon-europe-2019-thoughts-and-insights-by-airflow-committers/?utm_source=ApacheAirflowBlog&utm_medium=Npaid&utm_campaign=Blog&utm_term=Article&utm_content=AAB_NOP_BLG_ART_APC_001" target="_blank">insights on the Polidea blog</a>.</p>
+<p>In the past three and a half months, we have updated everything: created a more efficient landing page,
+enhanced information architecture, and improved UX & UI. Most importantly, the website now has capabilities
+to be translated into many languages. This is our effort to foster a more inclusive community around
+Apache Airflow, and we look forward to seeing contributions in Spanish, Chinese, Russian, and other languages as well!</p>
+
+<p>We built our website on Docsy, a platform that is easy to use and contribute to. Follow
+<a href="https://github.com/apache/airflow-site/blob/aip-11/README.md" target="_blank">these steps</a> to set up your environment and
+to create your first pull request. You may also use
+the new website for your own open source project as a template.
+All of our <a href="https://github.com/apache/airflow-site/tree/aip-11" target="_blank">code is open and hosted on Github</a>.</p>
+
+<p>Share your questions, comments, and suggestions with us, to help us improve the website.
+We hope that this new design makes finding documentation about Airflow easier,
+and that its improved accessibility increases adoption and use of Apache Airflow around the world.</p>
+
+<p>Happy browsing!</p>
</div>
@@ -459,16 +479,16 @@ if (!doNotTrack) {
<div class="pager">
- <a href="/blog/its-a-breeze-to-develop-apache-airflow/">
+ <a href="/blog/apache-con-europe-2019-thoughts-and-insights-by-airflow-commiters/">
<button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" >Previous</button>
</a>
- <a >
+ <a href="/blog/airflow-survey/">
-<button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" disabled>Next</button>
+<button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" >Next</button>
</a>
</div>
@@ -481,6 +501,46 @@ if (!doNotTrack) {
+ <div class="blog-pager">
+ <h5 class="header__xsmall--greyish-brown">Read also</h5>
+ <div class="pager">
+ <div class="list-items">
+
+ <div class="list-item list-item--wide">
+
+
+<div class="card">
+ <div class="box-event__blogpost">
+ <div class="box-event__blogpost--metadata">
+ <div class="tags-container">
+
+
+ <a class="tag" href="/blog/tags/community/">Community</a>
+
+
+ </div>
+ <span class="bodytext__medium--brownish-grey box-event__blogpost--date">Fri, Nov 22, 2019</span>
+ </div>
+ <p class="box-event__blogpost--header">ApacheCon Europe 2019 — Thoughts and Insights by Airflow Commiters</p>
+ <p class="box-event__blogpost--author">Polidea</p>
+ <p class="box-event__blogpost--description">Here come some thoughts by Airflow committers and contributors from the ApacheCon Europe 2019. Get to know the ASF community!</p>
+ <div class="mt-auto">
+ <a href="/blog/apache-con-europe-2019-thoughts-and-insights-by-airflow-commiters/">
+
+
+<button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" >Read more</button>
+
+ </a>
+ </div>
+ </div>
+</div>
+
+ </div>
+
+ </div>
+ </div>
+ </div>
+
</div>
@@ -489,7 +549,7 @@ if (!doNotTrack) {
<div class="base-layout--button">
- <a href=https://github.com/apache/airflow-site/edit/aip-11/landing-pages/site/content/en/blog/apache-con-europe-2019-thoughts-and-insights-by-airflow-commiters.md>
+ <a href=https://github.com/apache/airflow-site/edit/aip-11/landing-pages/site/content/en/blog/announcing-new-website.md>
@@ -667,6 +727,6 @@ if (!doNotTrack) {
</body>
- <script src="/main.b2992.js"></script>
+ <script src="/main.e3a02.js"></script>
</html>
diff --git a/blog/apache-con-europe-2019-thoughts-and-insights-by-airflow-commiters/index.html b/blog/apache-con-europe-2019-thoughts-and-insights-by-airflow-commiters/index.html
index 93d8820..7cda438 100644
--- a/blog/apache-con-europe-2019-thoughts-and-insights-by-airflow-commiters/index.html
+++ b/blog/apache-con-europe-2019-thoughts-and-insights-by-airflow-commiters/index.html
@@ -85,15 +85,15 @@ if (!doNotTrack) {
- <link rel="preload" href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" as="style">
- <link href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" rel="stylesheet" integrity="">
+ <link rel="preload" href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" as="style">
+ <link href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" rel="stylesheet" integrity="">
-<link rel="preload" href="/main.b2992.js" as="script">
-<link rel="preload" href="/chunk-1.b2992.js" as="script">
+<link rel="preload" href="/main.e3a02.js" as="script">
+<link rel="preload" href="/chunk-1.e3a02.js" as="script">
-<link rel="preload" href="/chunk-4.b2992.js" as="script">
+<link rel="preload" href="/chunk-4.e3a02.js" as="script">
@@ -465,10 +465,10 @@ if (!doNotTrack) {
<button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" >Previous</button>
</a>
- <a >
+ <a href="/blog/announcing-new-website/">
-<button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" disabled>Next</button>
+<button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" >Next</button>
</a>
</div>
@@ -481,6 +481,46 @@ if (!doNotTrack) {
+ <div class="blog-pager">
+ <h5 class="header__xsmall--greyish-brown">Read also</h5>
+ <div class="pager">
+ <div class="list-items">
+
+ <div class="list-item list-item--wide">
+
+
+<div class="card">
+ <div class="box-event__blogpost">
+ <div class="box-event__blogpost--metadata">
+ <div class="tags-container">
+
+
+ <a class="tag" href="/blog/tags/community/">Community</a>
+
+
+ </div>
+ <span class="bodytext__medium--brownish-grey box-event__blogpost--date">Wed, Dec 11, 2019</span>
+ </div>
+ <p class="box-event__blogpost--header">New Airflow website</p>
+ <p class="box-event__blogpost--author">Aizhamal Nurmamat kyzy</p>
+ <p class="box-event__blogpost--description">We are thrilled about our new website!</p>
+ <div class="mt-auto">
+ <a href="/blog/announcing-new-website/">
+
+
+<button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" >Read more</button>
+
+ </a>
+ </div>
+ </div>
+</div>
+
+ </div>
+
+ </div>
+ </div>
+ </div>
+
</div>
@@ -667,6 +707,6 @@ if (!doNotTrack) {
</body>
- <script src="/main.b2992.js"></script>
+ <script src="/main.e3a02.js"></script>
</html>
diff --git a/blog/index.html b/blog/index.html
index cb8d866..9af5c02 100644
--- a/blog/index.html
+++ b/blog/index.html
@@ -77,15 +77,15 @@ if (!doNotTrack) {
- <link rel="preload" href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" as="style">
- <link href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" rel="stylesheet" integrity="">
+ <link rel="preload" href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" as="style">
+ <link href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" rel="stylesheet" integrity="">
-<link rel="preload" href="/main.b2992.js" as="script">
-<link rel="preload" href="/chunk-1.b2992.js" as="script">
+<link rel="preload" href="/main.e3a02.js" as="script">
+<link rel="preload" href="/chunk-1.e3a02.js" as="script">
-<link rel="preload" href="/chunk-4.b2992.js" as="script">
+<link rel="preload" href="/chunk-4.e3a02.js" as="script">
@@ -391,6 +391,14 @@ if (!doNotTrack) {
<a class="tag" href="/blog/tags/development/">Development</a>
+
+ <a class="tag" href="/blog/tags/survey/">Survey</a>
+
+
+
+ <a class="tag" href="/blog/tags/users/">Users</a>
+
+
</div>
</div>
<div class="blog__list-items-wrapper">
@@ -411,6 +419,76 @@ if (!doNotTrack) {
<a class="tag" href="/blog/tags/community/">Community</a>
+
+ <a class="tag" href="/blog/tags/survey/">Survey</a>
+
+
+
+ <a class="tag" href="/blog/tags/users/">Users</a>
+
+
+ </div>
+ <span class="bodytext__medium--brownish-grey box-event__blogpost--date">Wed, Dec 11, 2019</span>
+ </div>
+ <p class="box-event__blogpost--header">Airflow Survey 2019</p>
+ <p class="box-event__blogpost--author">Tomek Urbaszek</p>
+ <p class="box-event__blogpost--description">Receiving and adjusting to our users’ feedback is a must. Let’s see who Airflow users are, how they play with it, and what they miss.</p>
+ <div class="mt-auto">
+ <a href="/blog/airflow-survey/">
+
+
+<button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" >Read more</button>
+
+ </a>
+ </div>
+ </div>
+</div>
+
+ </div>
+
+ <div class="list-item list-item--wide">
+
+
+<div class="card">
+ <div class="box-event__blogpost">
+ <div class="box-event__blogpost--metadata">
+ <div class="tags-container">
+
+
+ <a class="tag" href="/blog/tags/community/">Community</a>
+
+
+ </div>
+ <span class="bodytext__medium--brownish-grey box-event__blogpost--date">Wed, Dec 11, 2019</span>
+ </div>
+ <p class="box-event__blogpost--header">New Airflow website</p>
+ <p class="box-event__blogpost--author">Aizhamal Nurmamat kyzy</p>
+ <p class="box-event__blogpost--description">We are thrilled about our new website!</p>
+ <div class="mt-auto">
+ <a href="/blog/announcing-new-website/">
+
+
+<button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" >Read more</button>
+
+ </a>
+ </div>
+ </div>
+</div>
+
+ </div>
+
+ <div class="list-item list-item--wide">
+
+
+<div class="card">
+ <div class="box-event__blogpost">
+ <div class="box-event__blogpost--metadata">
+ <div class="tags-container">
+
+
+ <a class="tag" href="/blog/tags/community/">Community</a>
+
+
</div>
<span class="bodytext__medium--brownish-grey box-event__blogpost--date">Fri, Nov 22, 2019</span>
</div>
@@ -648,6 +726,6 @@ if (!doNotTrack) {
</body>
- <script src="/main.b2992.js"></script>
+ <script src="/main.e3a02.js"></script>
</html>
diff --git a/blog/index.xml b/blog/index.xml
index 1bbe152..ba176c4 100644
--- a/blog/index.xml
+++ b/blog/index.xml
@@ -14,6 +14,1032 @@
<item>
+ <title>Blog: Airflow Survey 2019</title>
+ <link>/blog/airflow-survey/</link>
+ <pubDate>Wed, 11 Dec 2019 00:00:00 +0000</pubDate>
+
+ <guid>/blog/airflow-survey/</guid>
+ <description>
+
+
+
+
+<h1 id="apache-airflow-survey-2019">Apache Airflow Survey 2019</h1>
+
+<p>Apache Airflow is <a href="https://www.astronomer.io/blog/why-airflow/" target="_blank">growing faster than ever</a>.
+Thus, receiving and adjusting to our users’ feedback is a must. We created
+<a href="https://forms.gle/XAzR1pQBZiftvPQM7" target="_blank">survey</a> and we got <strong>308</strong> responses.
+Let’s see who Airflow users are, how they play with it, and what they miss.</p>
+
+<h1 id="overview-of-the-user">Overview of the user</h1>
+
+<p><strong>What best describes your current occupation?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Data Engineer</td>
+<td>194</td>
+<td>62.99%</td>
+</tr>
+
+<tr>
+<td>Developer</td>
+<td>34</td>
+<td>11.04%</td>
+</tr>
+
+<tr>
+<td>Architect</td>
+<td>23</td>
+<td>7.47%</td>
+</tr>
+
+<tr>
+<td>Data Scientist</td>
+<td>19</td>
+<td>6.17%</td>
+</tr>
+
+<tr>
+<td>Data Analyst</td>
+<td>13</td>
+<td>4.22%</td>
+</tr>
+
+<tr>
+<td>DevOps</td>
+<td>13</td>
+<td>4.22%</td>
+</tr>
+
+<tr>
+<td>IT Administrator</td>
+<td>2</td>
+<td>0.65%</td>
+</tr>
+
+<tr>
+<td>Machine Learning Engineer</td>
+<td>2</td>
+<td>0.65%</td>
+</tr>
+
+<tr>
+<td>Manager</td>
+<td>2</td>
+<td>0.65%</td>
+</tr>
+
+<tr>
+<td>Operations</td>
+<td>2</td>
+<td>0.65%</td>
+</tr>
+
+<tr>
+<td>Chief Data Officer</td>
+<td>1</td>
+<td>0.32%</td>
+</tr>
+
+<tr>
+<td>Engineering Manager</td>
+<td>1</td>
+<td>0.32%</td>
+</tr>
+
+<tr>
+<td>Intern</td>
+<td>1</td>
+<td>0.32%</td>
+</tr>
+
+<tr>
+<td>Product owner</td>
+<td>1</td>
+<td>0.32%</td>
+</tr>
+
+<tr>
+<td>Quant</td>
+<td>1</td>
+<td>0.32%</td>
+</tr>
+</tbody>
+</table>
+
+<p><strong>In your day to day job, what do you use Airflow for?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Data processing (ETL)</td>
+<td>298</td>
+<td>96.75%</td>
+</tr>
+
+<tr>
+<td>Artificial Intelligence and Machine Learning Pipelines</td>
+<td>90</td>
+<td>29.22%</td>
+</tr>
+
+<tr>
+<td>Automating DevOps operations</td>
+<td>64</td>
+<td>20.78%</td>
+</tr>
+</tbody>
+</table>
+
+<p>According to the survey, most of the Airflow users are the “data” people. Moreover,
+28.57% uses Airflow to both ETL and ML pipelines meaning that those two fields
+are somehow connected. Only five respondents use Airflow for DevOps operations only,
+That means that other 59 people who use Airflow for DevOps stuff use it also for
+ETL / ML purposes.</p>
+
+<p><strong>How many active DAGs do you have in your largest Airflow instance?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>0-20</td>
+<td>115</td>
+<td>37.34%</td>
+</tr>
+
+<tr>
+<td>21-40</td>
+<td>65</td>
+<td>21.10%</td>
+</tr>
+
+<tr>
+<td>41-60</td>
+<td>44</td>
+<td>14.29%</td>
+</tr>
+
+<tr>
+<td>61-100</td>
+<td>28</td>
+<td>9.09%</td>
+</tr>
+
+<tr>
+<td>101-200</td>
+<td>28</td>
+<td>9.09%</td>
+</tr>
+
+<tr>
+<td>201-300</td>
+<td>7</td>
+<td>2.27%</td>
+</tr>
+
+<tr>
+<td>301-999</td>
+<td>8</td>
+<td>2.60%</td>
+</tr>
+
+<tr>
+<td>1000+</td>
+<td>13</td>
+<td>4.22%</td>
+</tr>
+</tbody>
+</table>
+
+<p>The majority of users do not exceed 100 active DAGs per Airflow instance. However,
+as we can see there are users who exceed thousands of DAGs with a maximum number 5000.</p>
+
+<p><strong>What is the maximum number of tasks that you have used in one DAG?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>0-10</td>
+<td>61</td>
+<td>19.81%</td>
+</tr>
+
+<tr>
+<td>11-20</td>
+<td>60</td>
+<td>19.48%</td>
+</tr>
+
+<tr>
+<td>21-30</td>
+<td>31</td>
+<td>10.06%</td>
+</tr>
+
+<tr>
+<td>31-40</td>
+<td>21</td>
+<td>6.82%</td>
+</tr>
+
+<tr>
+<td>41-50</td>
+<td>26</td>
+<td>8.44%</td>
+</tr>
+
+<tr>
+<td>51-100</td>
+<td>36</td>
+<td>11.69%</td>
+</tr>
+
+<tr>
+<td>101-200</td>
+<td>28</td>
+<td>9.09%</td>
+</tr>
+
+<tr>
+<td>201-500</td>
+<td>21</td>
+<td>6.82%</td>
+</tr>
+
+<tr>
+<td>501+</td>
+<td>24</td>
+<td>11.54%</td>
+</tr>
+</tbody>
+</table>
+
+<p>The given maximum number of tasks in a single DAG was 10 000 (!). The number of tasks
+depends on the purposes of a DAG, so it’s rather hard to say if users have “simple”
+or “complicated” workflows.</p>
+
+<p><strong>When onboarding new members to Airflow, what is the biggest problem?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>No guide on best practises on developing DAGs</td>
+<td>160</td>
+<td>51.95%</td>
+</tr>
+
+<tr>
+<td>Small number of tutorials on different aspects of using Airflow</td>
+<td>57</td>
+<td>18.51%</td>
+</tr>
+
+<tr>
+<td>Documentation is not clear enough</td>
+<td>42</td>
+<td>13.64%</td>
+</tr>
+
+<tr>
+<td>Small number of blogs regarding Airflow</td>
+<td>6</td>
+<td>1.95%</td>
+</tr>
+
+<tr>
+<td>Other</td>
+<td>43</td>
+<td>13.96%</td>
+</tr>
+</tbody>
+</table>
+
+<p>This is an important result. Using Airflow is all about writing and scheduling DAGs.
+No guide or any other complete resource on best practices for developing Dags is a big
+problem. Diving deep in the “other” answers, we can find that:</p>
+
+<ul>
+<li>Airflow’s “magic” (scheduler, executors, schedule times) is hard to understand</li>
+<li>DAG testing is not easy to do and to explain</li>
+<li>Airflow UI needs some love.</li>
+</ul>
+
+<p><strong>How likely are you to recommend Apache Airflow?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Very Likely</td>
+<td>140</td>
+<td>45.45%</td>
+</tr>
+
+<tr>
+<td>Likely</td>
+<td>124</td>
+<td>40.26%</td>
+</tr>
+
+<tr>
+<td>Neutral</td>
+<td>33</td>
+<td>10.71%</td>
+</tr>
+
+<tr>
+<td>Unlikely</td>
+<td>8</td>
+<td>2.60%</td>
+</tr>
+
+<tr>
+<td>Very unlikely</td>
+<td>3</td>
+<td>0.97%</td>
+</tr>
+</tbody>
+</table>
+
+<p>This means that more than 85% of people who use Airflow like it. It seems Airflow does
+its job nicely. However, we have to remember that this survey is likely biased - it’s
+more likely that you respond to the survey if you like the tool you use. Should we
+focus then on those 11 people who did not like Airflow? It’s a good question.</p>
+
+<h2 id="airflow-usage">Airflow usage</h2>
+
+<p><strong>Which interface(s) of Airflow do you use as part of your current role?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Original Airflow Graphical User Interface</td>
+<td>297</td>
+<td>96.43%</td>
+</tr>
+
+<tr>
+<td>CLI</td>
+<td>126</td>
+<td>40.91%</td>
+</tr>
+
+<tr>
+<td>Original Airflow Graphical User Interface, CLI</td>
+<td>117</td>
+<td>37.99%</td>
+</tr>
+
+<tr>
+<td>API</td>
+<td>60</td>
+<td>19.48%</td>
+</tr>
+
+<tr>
+<td>Original Airflow Graphical User Interface, CLI, API</td>
+<td>32</td>
+<td>10.39%</td>
+</tr>
+
+<tr>
+<td>Custom (own created) Airflow Graphical User Interface</td>
+<td>25</td>
+<td>8.12%</td>
+</tr>
+</tbody>
+</table>
+
+<p>It’s visible that usage of CLI goes in pair with using Airflow web UI. Our
+survey included some UX related questions to allow us to understand how users
+use Airflow webserver.</p>
+
+<p><strong>What do you use the Graphical User Interface for?</strong></p>
+
+<p><img src="plot1.png" alt="" /></p>
+
+<p><strong>What do you use CLI for?</strong></p>
+
+<p><img src="plot2.png" alt="" /></p>
+
+<p><strong>In Airflow, which UI view(s) are important for you?</strong></p>
+
+<p><img src="plot3.png" alt="" /></p>
+
+<p>Here we see that the majority uses Web UI mostly for monitoring purposes:</p>
+
+<ul>
+<li>Monitoring DAGs</li>
+<li>Accessing logs</li>
+</ul>
+
+<p>An interesting result is that many people seem not to use backfilling as
+there’s no other way than to do it by CLI.</p>
+
+<p><strong>What executor type do you use?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Celery</td>
+<td>138</td>
+<td>44.81%</td>
+</tr>
+
+<tr>
+<td>Local</td>
+<td>85</td>
+<td>27.60%</td>
+</tr>
+
+<tr>
+<td>Kubernetes</td>
+<td>52</td>
+<td>16.88%</td>
+</tr>
+
+<tr>
+<td>Sequential</td>
+<td>22</td>
+<td>7.14%</td>
+</tr>
+
+<tr>
+<td>Other</td>
+<td>11</td>
+<td>3.57</td>
+</tr>
+</tbody>
+</table>
+
+<p>The other option mostly consisted of information that someone uses a few types or is
+migrating from one executor to another. What can be observed is an increase in usage
+of Local and Kubernetes executors when compared to results from an earlier <a href="https://ash.berlintaylor.com/writings/2019/02/airflow-user-survey-2019/" target="_blank">survey done
+by Ash</a>.</p>
+
+<p><strong>Do you use Kubernetes-based deployments for Airflow?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>No - we do not plan to use Kubernetes near term</td>
+<td>88</td>
+<td>28.57%</td>
+</tr>
+
+<tr>
+<td>Yes - setup on our own via Helm Chart or similar</td>
+<td>65</td>
+<td>21.10%</td>
+</tr>
+
+<tr>
+<td>Not yet - but we use Kubernetes in our organization and we could move</td>
+<td>61</td>
+<td>19.81%</td>
+</tr>
+
+<tr>
+<td>Yes - via managed service in the cloud (Composer / Astronomer etc.)</td>
+<td>45</td>
+<td>14.61%</td>
+</tr>
+
+<tr>
+<td>Not yet - but we plan to deploy Kubernetes in our organization soon</td>
+<td>42</td>
+<td>13.64%</td>
+</tr>
+
+<tr>
+<td>Other</td>
+<td>7</td>
+<td>2.27%</td>
+</tr>
+</tbody>
+</table>
+
+<p>The most interesting thing is that there’s nearly 30% of users who do not use Kubernetes,
+and they are not going to move. This means we should keep other deployment options in
+mind when working on Airflow 2.0. On the other hand, almost 70% of the users already
+use Kubernetes, or it’s a viable option for them.</p>
+
+<p><strong>Do you combine multiple DAGs?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>No, I don&rsquo;t combine multiple DAGs</td>
+<td>127</td>
+<td>41.23%</td>
+</tr>
+
+<tr>
+<td>Yes, through SubDAG</td>
+<td>73</td>
+<td>23.70%</td>
+</tr>
+
+<tr>
+<td>Yes, by triggering another DAG</td>
+<td>72</td>
+<td>23.38%</td>
+</tr>
+
+<tr>
+<td>Other</td>
+<td>36</td>
+<td>11.69%</td>
+</tr>
+</tbody>
+</table>
+
+<p>In the other category, 9 people explicitly mentioned using <code>ExternalTaskSensor</code>,
+and I think it could be treated as running subDAGs by triggering other DAGs.</p>
+
+<p><strong>Do you use Airflow Plugins? If yes, what do you use it for?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Adding new operators/sensors and hooks</td>
+<td>187</td>
+<td>60.71%</td>
+</tr>
+
+<tr>
+<td>I don&rsquo;t use Airflow plugins</td>
+<td>109</td>
+<td>35.39%</td>
+</tr>
+
+<tr>
+<td>Adding AppBuilder views &amp; menu items</td>
+<td>31</td>
+<td>10.06%</td>
+</tr>
+
+<tr>
+<td>Adding new executor</td>
+<td>18</td>
+<td>5.84%</td>
+</tr>
+
+<tr>
+<td>Adding OperatorExtraLinks</td>
+<td>7</td>
+<td>2.27%</td>
+</tr>
+</tbody>
+</table>
+
+<p>The high percentage - 60% for “Adding new operators/sensors and hooks” is quite a
+surprising result for some of us - especially that you do not actually need to use the
+plugin mechanism to add any of those. Those are standard python objects, and you can
+simply drop your hooks/operators/sensors code to <code>PYTHONPATH</code> environment variable and
+they will work. It seems that this may be a result of a lack of best practices guide.</p>
+
+<p>Plugins are more useful for adding views and menu items - yet only 10%.
+OperatorExtraLinks are even more useful (though relatively new) feature, so it’s not
+entirely surprising they are hardly used.</p>
+
+<p>It was also kind of surprising that someone at all uses plugins to use their own
+executors. We considered removing that option recently - but now we have to rethink
+our approach.</p>
+
+<p><strong>What metrics do you use to monitor Airflow?</strong></p>
+
+<p>There were a lot of different responses. Some use Prometheus and other services,
+others do not use any monitoring. One of the interesting responses linked to this
+solution for <a href="https://github.com/mastak/airflow_operators_metrics" target="_blank">airflow_operators_metrics</a>.</p>
+
+<h2 id="external-services">External services</h2>
+
+<p><strong>What external services do you use in your Airflow DAGs?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Amazon Web Services</td>
+<td>160</td>
+<td>51.95%</td>
+</tr>
+
+<tr>
+<td>Internal company systems</td>
+<td>150</td>
+<td>48.7%</td>
+</tr>
+
+<tr>
+<td>Hadoop / Spark / Flink / Other Apache software</td>
+<td>119</td>
+<td>38.64%</td>
+</tr>
+
+<tr>
+<td>Google Cloud Platform / Google APIs</td>
+<td>112</td>
+<td>36.36%</td>
+</tr>
+
+<tr>
+<td>Microsoft Azure</td>
+<td>28</td>
+<td>9.09%</td>
+</tr>
+
+<tr>
+<td>I do not use external services in my Airflow DAGs</td>
+<td>18</td>
+<td>5.84%</td>
+</tr>
+</tbody>
+</table>
+
+<p>It’s not surprising that Amazon Web Services is leading the way as they are considered the most mature
+cloud provider. Internal system and other Apache products on the next two positions are
+quite understandable if we take into account that the majority uses Airflow for ETL processes.</p>
+
+<p><strong>What external services do you use in your Airflow DAGs? (Mixed providers)</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Google Cloud Platform / Google APIs, Amazon Web Services</td>
+<td>44</td>
+<td>14.29%</td>
+</tr>
+
+<tr>
+<td>Amazon Web Services, Microsoft Azure</td>
+<td>5</td>
+<td>1.62%</td>
+</tr>
+
+<tr>
+<td>Google Cloud Platform / Google APIs, Microsoft Azure</td>
+<td>4</td>
+<td>1.3%</td>
+</tr>
+</tbody>
+</table>
+
+<p>This result is not surprising because companies usually prefer to stick with one cloud
+provider.</p>
+
+<p><strong>How do you integrate with external services?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Using Bash / Python operator</td>
+<td>220</td>
+<td>71.43%</td>
+</tr>
+
+<tr>
+<td>Using existing, dedicated operators / hooks</td>
+<td>217</td>
+<td>70.45%</td>
+</tr>
+
+<tr>
+<td>Using own, custom operators / hooks</td>
+<td>216</td>
+<td>70.13%</td>
+</tr>
+</tbody>
+</table>
+
+<p>We had some anecdotal evidence that people use more Python/Bash operators than the
+dedicated ones - but it looks like all ways of using Airflow to connect to external
+services are equally popular.</p>
+
+<h2 id="what-can-be-improved">What can be improved</h2>
+
+<p><strong>In your opinion, what could be improved in Airflow?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Scheduler performance</td>
+<td>189</td>
+<td>61.36%</td>
+</tr>
+
+<tr>
+<td>Web UI</td>
+<td>180</td>
+<td>58.44%</td>
+</tr>
+
+<tr>
+<td>Logging, monitoring and alerting</td>
+<td>145</td>
+<td>47.08%</td>
+</tr>
+
+<tr>
+<td>Examples, how-to, onboarding documentation</td>
+<td>143</td>
+<td>46.43%</td>
+</tr>
+
+<tr>
+<td>Technical documentation</td>
+<td>137</td>
+<td>44.48%</td>
+</tr>
+
+<tr>
+<td>Reliability</td>
+<td>112</td>
+<td>36.36%</td>
+</tr>
+
+<tr>
+<td>REST API</td>
+<td>96</td>
+<td>31.17%</td>
+</tr>
+
+<tr>
+<td>Authentication and authorization</td>
+<td>89</td>
+<td>28.9%</td>
+</tr>
+
+<tr>
+<td>External integration e.g. AWS, GCP, Apache product</td>
+<td>49</td>
+<td>15.91%</td>
+</tr>
+
+<tr>
+<td>CLI</td>
+<td>41</td>
+<td>13.31%</td>
+</tr>
+
+<tr>
+<td>I don’t know</td>
+<td>5</td>
+<td>1.62%</td>
+</tr>
+</tbody>
+</table>
+
+<p>The results are rather quite self-explaining. Improved performance of Airflow, better
+UI, and more telemetry are desirable. But this should go in pair with improved
+documentation and resources about using the Airflow, especially when we
+take into account the problem of onboarding new users.</p>
+
+<p>Another interesting point from that question is that only 16% think that operators
+should be extended and improved. This suggests that we should focus on improving
+Airflow core instead of adding more and more integrations.</p>
+
+<p><strong>What would be the most interesting feature for you?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Production-ready Airflow docker image</td>
+<td>175</td>
+<td>56.82%</td>
+</tr>
+
+<tr>
+<td>Declarative way of writing DAGs / automated DAGs generation</td>
+<td>155</td>
+<td>50.32%</td>
+</tr>
+
+<tr>
+<td>Horizontal Autoscaling</td>
+<td>122</td>
+<td>39.61%</td>
+</tr>
+
+<tr>
+<td>Asynchronous Operators</td>
+<td>97</td>
+<td>31.49%</td>
+</tr>
+
+<tr>
+<td>Stateless web server</td>
+<td>81</td>
+<td>26.3%</td>
+</tr>
+
+<tr>
+<td>Knative Executor</td>
+<td>48</td>
+<td>15.58%</td>
+</tr>
+
+<tr>
+<td>I already have all I need</td>
+<td>13</td>
+<td>4.22%</td>
+</tr>
+</tbody>
+</table>
+
+<p>Production Docker image wins, and it’s not a surprise. We all know that deploying
+Airflow is not a plug and play process, and that’s why the official image is being
+worked on by Jarek Potiuk. An unexpected result is that half of the users would like to
+have a declarative way of creating DAGs. That seems to be something that is “against Airflow”
+as we always emphasize the possibility of writing workflows in pure python. Stories
+about DAG generators are not new and confirm that there’s a need for a way to
+declare DAGs.</p>
+
+<h2 id="data">Data</h2>
+
+<p>If you think I missed something and you want to look for insights on your own the data is available
+for you here:</p>
+
+<ul>
+<li>Original data: <a href="https://storage.googleapis.com/airflow-survey/survey.csv" target="_blank">https://storage.googleapis.com/airflow-survey/survey.csv</a></li>
+<li>Processed: <a href="https://storage.googleapis.com/airflow-survey/airflow_survey_processed.csv" target="_blank">https://storage.googleapis.com/airflow-survey/airflow_survey_processed.csv</a></li>
+</ul>
+
+<p>The processed data includes multi-choice options one-hot encoded. If you find any interesting
+insight, please update the article (<a href="https://github.com/apache/airflow-site/blob/aip-11/CONTRIBUTE.md" target="_blank">make PR</a>
+to Airflow site).</p>
+
+ </description>
+ </item>
+
+ <item>
+ <title>Blog: New Airflow website</title>
+ <link>/blog/announcing-new-website/</link>
+ <pubDate>Wed, 11 Dec 2019 00:00:00 +0000</pubDate>
+
+ <guid>/blog/announcing-new-website/</guid>
+ <description>
+
+
+ <p>The brand <a href="https://airflow.apache.org/" target="_blank">new Airflow website</a> has arrived! Those who have been following the process know that the journey to update <a href="https://airflow.readthedocs.io/en/1.10.6/" target="_blank">the old Airflow website</a> started at the beginning of the year.
+Thanks to sponsorship from the Cloud Composer team at Google that allowed us to
+collaborate with <a href="https://www.polidea.com/" target="_blank">Polidea</a> and with their design studio <a href="https://utilodesign.com/" target="_blank">Utilo</a>, and deliver an awesome website.</p>
+
+<p>Documentation of open source projects is key to engaging new contributors in the maintenance,
+development, and adoption of software. We want the Apache Airflow community to have
+the best possible experience to contribute and use the project. We also took this opportunity to make the project
+more accessible, and in doing so, increase its reach.</p>
+
+<p>In the past three and a half months, we have updated everything: created a more efficient landing page,
+enhanced information architecture, and improved UX &amp; UI. Most importantly, the website now has capabilities
+to be translated into many languages. This is our effort to foster a more inclusive community around
+Apache Airflow, and we look forward to seeing contributions in Spanish, Chinese, Russian, and other languages as well!</p>
+
+<p>We built our website on Docsy, a platform that is easy to use and contribute to. Follow
+<a href="https://github.com/apache/airflow-site/blob/aip-11/README.md" target="_blank">these steps</a> to set up your environment and
+to create your first pull request. You may also use
+the new website for your own open source project as a template.
+All of our <a href="https://github.com/apache/airflow-site/tree/aip-11" target="_blank">code is open and hosted on Github</a>.</p>
+
+<p>Share your questions, comments, and suggestions with us, to help us improve the website.
+We hope that this new design makes finding documentation about Airflow easier,
+and that its improved accessibility increases adoption and use of Apache Airflow around the world.</p>
+
+<p>Happy browsing!</p>
+
+ </description>
+ </item>
+
+ <item>
<title>Blog: ApacheCon Europe 2019 — Thoughts and Insights by Airflow Commiters</title>
<link>/blog/apache-con-europe-2019-thoughts-and-insights-by-airflow-commiters/</link>
<pubDate>Fri, 22 Nov 2019 00:00:00 +0000</pubDate>
diff --git a/blog/its-a-breeze-to-develop-apache-airflow/index.html b/blog/its-a-breeze-to-develop-apache-airflow/index.html
index 6729bfb..5ae2b8a 100644
--- a/blog/its-a-breeze-to-develop-apache-airflow/index.html
+++ b/blog/its-a-breeze-to-develop-apache-airflow/index.html
@@ -85,15 +85,15 @@ if (!doNotTrack) {
- <link rel="preload" href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" as="style">
- <link href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" rel="stylesheet" integrity="">
+ <link rel="preload" href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" as="style">
+ <link href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" rel="stylesheet" integrity="">
-<link rel="preload" href="/main.b2992.js" as="script">
-<link rel="preload" href="/chunk-1.b2992.js" as="script">
+<link rel="preload" href="/main.e3a02.js" as="script">
+<link rel="preload" href="/chunk-1.e3a02.js" as="script">
-<link rel="preload" href="/chunk-4.b2992.js" as="script">
+<link rel="preload" href="/chunk-4.e3a02.js" as="script">
@@ -706,6 +706,6 @@ if (!doNotTrack) {
</body>
- <script src="/main.b2992.js"></script>
+ <script src="/main.e3a02.js"></script>
</html>
diff --git a/blog/tags/community/index.html b/blog/tags/community/index.html
index fb899ac..08fb3c0 100644
--- a/blog/tags/community/index.html
+++ b/blog/tags/community/index.html
@@ -30,21 +30,21 @@
<meta name="msapplication-TileImage" content="/favicons/ms-icon-144x144.png">
<meta name="theme-color" content="#ffffff">
-<title>Community | Apache Airflow</title><meta property="og:title" content="Community" />
+<title>community | Apache Airflow</title><meta property="og:title" content="community" />
<meta property="og:description" content="Platform created by community to programmatically author, schedule and monitor workflows." />
<meta property="og:type" content="website" />
<meta property="og:url" content="/blog/tags/community/" />
<meta property="og:image" content="/images/feature-image.png" />
-<meta property="og:updated_time" content="2019-11-22T00:00:00+00:00" /><meta property="og:site_name" content="Apache Airflow" />
-<meta itemprop="name" content="Community">
+<meta property="og:updated_time" content="2019-12-11T00:00:00+00:00" /><meta property="og:site_name" content="Apache Airflow" />
+<meta itemprop="name" content="community">
<meta itemprop="description" content="Platform created by community to programmatically author, schedule and monitor workflows.">
<meta name="twitter:card" content="summary_large_image"/>
<meta name="twitter:image" content="/images/feature-image.png"/>
-<meta name="twitter:title" content="Community"/>
+<meta name="twitter:title" content="community"/>
<meta name="twitter:description" content="Platform created by community to programmatically author, schedule and monitor workflows."/>
@@ -78,15 +78,15 @@ if (!doNotTrack) {
- <link rel="preload" href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" as="style">
- <link href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" rel="stylesheet" integrity="">
+ <link rel="preload" href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" as="style">
+ <link href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" rel="stylesheet" integrity="">
-<link rel="preload" href="/main.b2992.js" as="script">
-<link rel="preload" href="/chunk-1.b2992.js" as="script">
+<link rel="preload" href="/main.e3a02.js" as="script">
+<link rel="preload" href="/chunk-1.e3a02.js" as="script">
-<link rel="preload" href="/chunk-4.b2992.js" as="script">
+<link rel="preload" href="/chunk-4.e3a02.js" as="script">
@@ -395,6 +395,16 @@ if (!doNotTrack) {
href="/blog/tags/development/">Development</a>
+
+ <a class="tag "
+ href="/blog/tags/survey/">Survey</a>
+
+
+
+ <a class="tag "
+ href="/blog/tags/users/">Users</a>
+
+
</div>
</div>
<div class="blog__list-items-wrapper">
@@ -415,6 +425,76 @@ if (!doNotTrack) {
<a class="tag" href="/blog/tags/community/">Community</a>
+
+ <a class="tag" href="/blog/tags/survey/">Survey</a>
+
+
+
+ <a class="tag" href="/blog/tags/users/">Users</a>
+
+
+ </div>
+ <span class="bodytext__medium--brownish-grey box-event__blogpost--date">Wed, Dec 11, 2019</span>
+ </div>
+ <p class="box-event__blogpost--header">Airflow Survey 2019</p>
+ <p class="box-event__blogpost--author">Tomek Urbaszek</p>
+ <p class="box-event__blogpost--description">Receiving and adjusting to our users’ feedback is a must. Let’s see who Airflow users are, how they play with it, and what they miss.</p>
+ <div class="mt-auto">
+ <a href="/blog/airflow-survey/">
+
+
+<button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" >Read more</button>
+
+ </a>
+ </div>
+ </div>
+</div>
+
+ </div>
+
+ <div class="list-item list-item--wide">
+
+
+<div class="card">
+ <div class="box-event__blogpost">
+ <div class="box-event__blogpost--metadata">
+ <div class="tags-container">
+
+
+ <a class="tag" href="/blog/tags/community/">Community</a>
+
+
+ </div>
+ <span class="bodytext__medium--brownish-grey box-event__blogpost--date">Wed, Dec 11, 2019</span>
+ </div>
+ <p class="box-event__blogpost--header">New Airflow website</p>
+ <p class="box-event__blogpost--author">Aizhamal Nurmamat kyzy</p>
+ <p class="box-event__blogpost--description">We are thrilled about our new website!</p>
+ <div class="mt-auto">
+ <a href="/blog/announcing-new-website/">
+
+
+<button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" >Read more</button>
+
+ </a>
+ </div>
+ </div>
+</div>
+
+ </div>
+
+ <div class="list-item list-item--wide">
+
+
+<div class="card">
+ <div class="box-event__blogpost">
+ <div class="box-event__blogpost--metadata">
+ <div class="tags-container">
+
+
+ <a class="tag" href="/blog/tags/community/">Community</a>
+
+
</div>
<span class="bodytext__medium--brownish-grey box-event__blogpost--date">Fri, Nov 22, 2019</span>
</div>
@@ -621,6 +701,6 @@ if (!doNotTrack) {
</body>
- <script src="/main.b2992.js"></script>
+ <script src="/main.e3a02.js"></script>
</html>
diff --git a/blog/tags/community/index.xml b/blog/tags/community/index.xml
index 6af15f8..4ace7b4 100644
--- a/blog/tags/community/index.xml
+++ b/blog/tags/community/index.xml
@@ -1,10 +1,10 @@
<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">
<channel>
- <title>Apache Airflow – Community</title>
+ <title>Apache Airflow – community</title>
<link>/blog/tags/community/</link>
- <description>Recent content in Community on Apache Airflow</description>
+ <description>Recent content in community on Apache Airflow</description>
<generator>Hugo -- gohugo.io</generator>
- <lastBuildDate>Fri, 22 Nov 2019 00:00:00 +0000</lastBuildDate>
+ <lastBuildDate>Wed, 11 Dec 2019 00:00:00 +0000</lastBuildDate>
<atom:link href="/blog/tags/community/index.xml" rel="self" type="application/rss+xml" />
@@ -15,6 +15,1032 @@
<item>
+ <title>Blog: Airflow Survey 2019</title>
+ <link>/blog/airflow-survey/</link>
+ <pubDate>Wed, 11 Dec 2019 00:00:00 +0000</pubDate>
+
+ <guid>/blog/airflow-survey/</guid>
+ <description>
+
+
+
+
+<h1 id="apache-airflow-survey-2019">Apache Airflow Survey 2019</h1>
+
+<p>Apache Airflow is <a href="https://www.astronomer.io/blog/why-airflow/" target="_blank">growing faster than ever</a>.
+Thus, receiving and adjusting to our users’ feedback is a must. We created
+<a href="https://forms.gle/XAzR1pQBZiftvPQM7" target="_blank">survey</a> and we got <strong>308</strong> responses.
+Let’s see who Airflow users are, how they play with it, and what they miss.</p>
+
+<h1 id="overview-of-the-user">Overview of the user</h1>
+
+<p><strong>What best describes your current occupation?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Data Engineer</td>
+<td>194</td>
+<td>62.99%</td>
+</tr>
+
+<tr>
+<td>Developer</td>
+<td>34</td>
+<td>11.04%</td>
+</tr>
+
+<tr>
+<td>Architect</td>
+<td>23</td>
+<td>7.47%</td>
+</tr>
+
+<tr>
+<td>Data Scientist</td>
+<td>19</td>
+<td>6.17%</td>
+</tr>
+
+<tr>
+<td>Data Analyst</td>
+<td>13</td>
+<td>4.22%</td>
+</tr>
+
+<tr>
+<td>DevOps</td>
+<td>13</td>
+<td>4.22%</td>
+</tr>
+
+<tr>
+<td>IT Administrator</td>
+<td>2</td>
+<td>0.65%</td>
+</tr>
+
+<tr>
+<td>Machine Learning Engineer</td>
+<td>2</td>
+<td>0.65%</td>
+</tr>
+
+<tr>
+<td>Manager</td>
+<td>2</td>
+<td>0.65%</td>
+</tr>
+
+<tr>
+<td>Operations</td>
+<td>2</td>
+<td>0.65%</td>
+</tr>
+
+<tr>
+<td>Chief Data Officer</td>
+<td>1</td>
+<td>0.32%</td>
+</tr>
+
+<tr>
+<td>Engineering Manager</td>
+<td>1</td>
+<td>0.32%</td>
+</tr>
+
+<tr>
+<td>Intern</td>
+<td>1</td>
+<td>0.32%</td>
+</tr>
+
+<tr>
+<td>Product owner</td>
+<td>1</td>
+<td>0.32%</td>
+</tr>
+
+<tr>
+<td>Quant</td>
+<td>1</td>
+<td>0.32%</td>
+</tr>
+</tbody>
+</table>
+
+<p><strong>In your day to day job, what do you use Airflow for?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Data processing (ETL)</td>
+<td>298</td>
+<td>96.75%</td>
+</tr>
+
+<tr>
+<td>Artificial Intelligence and Machine Learning Pipelines</td>
+<td>90</td>
+<td>29.22%</td>
+</tr>
+
+<tr>
+<td>Automating DevOps operations</td>
+<td>64</td>
+<td>20.78%</td>
+</tr>
+</tbody>
+</table>
+
+<p>According to the survey, most of the Airflow users are the “data” people. Moreover,
+28.57% uses Airflow to both ETL and ML pipelines meaning that those two fields
+are somehow connected. Only five respondents use Airflow for DevOps operations only,
+That means that other 59 people who use Airflow for DevOps stuff use it also for
+ETL / ML purposes.</p>
+
+<p><strong>How many active DAGs do you have in your largest Airflow instance?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>0-20</td>
+<td>115</td>
+<td>37.34%</td>
+</tr>
+
+<tr>
+<td>21-40</td>
+<td>65</td>
+<td>21.10%</td>
+</tr>
+
+<tr>
+<td>41-60</td>
+<td>44</td>
+<td>14.29%</td>
+</tr>
+
+<tr>
+<td>61-100</td>
+<td>28</td>
+<td>9.09%</td>
+</tr>
+
+<tr>
+<td>101-200</td>
+<td>28</td>
+<td>9.09%</td>
+</tr>
+
+<tr>
+<td>201-300</td>
+<td>7</td>
+<td>2.27%</td>
+</tr>
+
+<tr>
+<td>301-999</td>
+<td>8</td>
+<td>2.60%</td>
+</tr>
+
+<tr>
+<td>1000+</td>
+<td>13</td>
+<td>4.22%</td>
+</tr>
+</tbody>
+</table>
+
+<p>The majority of users do not exceed 100 active DAGs per Airflow instance. However,
+as we can see there are users who exceed thousands of DAGs with a maximum number 5000.</p>
+
+<p><strong>What is the maximum number of tasks that you have used in one DAG?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>0-10</td>
+<td>61</td>
+<td>19.81%</td>
+</tr>
+
+<tr>
+<td>11-20</td>
+<td>60</td>
+<td>19.48%</td>
+</tr>
+
+<tr>
+<td>21-30</td>
+<td>31</td>
+<td>10.06%</td>
+</tr>
+
+<tr>
+<td>31-40</td>
+<td>21</td>
+<td>6.82%</td>
+</tr>
+
+<tr>
+<td>41-50</td>
+<td>26</td>
+<td>8.44%</td>
+</tr>
+
+<tr>
+<td>51-100</td>
+<td>36</td>
+<td>11.69%</td>
+</tr>
+
+<tr>
+<td>101-200</td>
+<td>28</td>
+<td>9.09%</td>
+</tr>
+
+<tr>
+<td>201-500</td>
+<td>21</td>
+<td>6.82%</td>
+</tr>
+
+<tr>
+<td>501+</td>
+<td>24</td>
+<td>11.54%</td>
+</tr>
+</tbody>
+</table>
+
+<p>The given maximum number of tasks in a single DAG was 10 000 (!). The number of tasks
+depends on the purposes of a DAG, so it’s rather hard to say if users have “simple”
+or “complicated” workflows.</p>
+
+<p><strong>When onboarding new members to Airflow, what is the biggest problem?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>No guide on best practises on developing DAGs</td>
+<td>160</td>
+<td>51.95%</td>
+</tr>
+
+<tr>
+<td>Small number of tutorials on different aspects of using Airflow</td>
+<td>57</td>
+<td>18.51%</td>
+</tr>
+
+<tr>
+<td>Documentation is not clear enough</td>
+<td>42</td>
+<td>13.64%</td>
+</tr>
+
+<tr>
+<td>Small number of blogs regarding Airflow</td>
+<td>6</td>
+<td>1.95%</td>
+</tr>
+
+<tr>
+<td>Other</td>
+<td>43</td>
+<td>13.96%</td>
+</tr>
+</tbody>
+</table>
+
+<p>This is an important result. Using Airflow is all about writing and scheduling DAGs.
+No guide or any other complete resource on best practices for developing Dags is a big
+problem. Diving deep in the “other” answers, we can find that:</p>
+
+<ul>
+<li>Airflow’s “magic” (scheduler, executors, schedule times) is hard to understand</li>
+<li>DAG testing is not easy to do and to explain</li>
+<li>Airflow UI needs some love.</li>
+</ul>
+
+<p><strong>How likely are you to recommend Apache Airflow?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Very Likely</td>
+<td>140</td>
+<td>45.45%</td>
+</tr>
+
+<tr>
+<td>Likely</td>
+<td>124</td>
+<td>40.26%</td>
+</tr>
+
+<tr>
+<td>Neutral</td>
+<td>33</td>
+<td>10.71%</td>
+</tr>
+
+<tr>
+<td>Unlikely</td>
+<td>8</td>
+<td>2.60%</td>
+</tr>
+
+<tr>
+<td>Very unlikely</td>
+<td>3</td>
+<td>0.97%</td>
+</tr>
+</tbody>
+</table>
+
+<p>This means that more than 85% of people who use Airflow like it. It seems Airflow does
+its job nicely. However, we have to remember that this survey is likely biased - it’s
+more likely that you respond to the survey if you like the tool you use. Should we
+focus then on those 11 people who did not like Airflow? It’s a good question.</p>
+
+<h2 id="airflow-usage">Airflow usage</h2>
+
+<p><strong>Which interface(s) of Airflow do you use as part of your current role?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Original Airflow Graphical User Interface</td>
+<td>297</td>
+<td>96.43%</td>
+</tr>
+
+<tr>
+<td>CLI</td>
+<td>126</td>
+<td>40.91%</td>
+</tr>
+
+<tr>
+<td>Original Airflow Graphical User Interface, CLI</td>
+<td>117</td>
+<td>37.99%</td>
+</tr>
+
+<tr>
+<td>API</td>
+<td>60</td>
+<td>19.48%</td>
+</tr>
+
+<tr>
+<td>Original Airflow Graphical User Interface, CLI, API</td>
+<td>32</td>
+<td>10.39%</td>
+</tr>
+
+<tr>
+<td>Custom (own created) Airflow Graphical User Interface</td>
+<td>25</td>
+<td>8.12%</td>
+</tr>
+</tbody>
+</table>
+
+<p>It’s visible that usage of CLI goes in pair with using Airflow web UI. Our
+survey included some UX related questions to allow us to understand how users
+use Airflow webserver.</p>
+
+<p><strong>What do you use the Graphical User Interface for?</strong></p>
+
+<p><img src="plot1.png" alt="" /></p>
+
+<p><strong>What do you use CLI for?</strong></p>
+
+<p><img src="plot2.png" alt="" /></p>
+
+<p><strong>In Airflow, which UI view(s) are important for you?</strong></p>
+
+<p><img src="plot3.png" alt="" /></p>
+
+<p>Here we see that the majority uses Web UI mostly for monitoring purposes:</p>
+
+<ul>
+<li>Monitoring DAGs</li>
+<li>Accessing logs</li>
+</ul>
+
+<p>An interesting result is that many people seem not to use backfilling as
+there’s no other way than to do it by CLI.</p>
+
+<p><strong>What executor type do you use?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Celery</td>
+<td>138</td>
+<td>44.81%</td>
+</tr>
+
+<tr>
+<td>Local</td>
+<td>85</td>
+<td>27.60%</td>
+</tr>
+
+<tr>
+<td>Kubernetes</td>
+<td>52</td>
+<td>16.88%</td>
+</tr>
+
+<tr>
+<td>Sequential</td>
+<td>22</td>
+<td>7.14%</td>
+</tr>
+
+<tr>
+<td>Other</td>
+<td>11</td>
+<td>3.57</td>
+</tr>
+</tbody>
+</table>
+
+<p>The other option mostly consisted of information that someone uses a few types or is
+migrating from one executor to another. What can be observed is an increase in usage
+of Local and Kubernetes executors when compared to results from an earlier <a href="https://ash.berlintaylor.com/writings/2019/02/airflow-user-survey-2019/" target="_blank">survey done
+by Ash</a>.</p>
+
+<p><strong>Do you use Kubernetes-based deployments for Airflow?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>No - we do not plan to use Kubernetes near term</td>
+<td>88</td>
+<td>28.57%</td>
+</tr>
+
+<tr>
+<td>Yes - setup on our own via Helm Chart or similar</td>
+<td>65</td>
+<td>21.10%</td>
+</tr>
+
+<tr>
+<td>Not yet - but we use Kubernetes in our organization and we could move</td>
+<td>61</td>
+<td>19.81%</td>
+</tr>
+
+<tr>
+<td>Yes - via managed service in the cloud (Composer / Astronomer etc.)</td>
+<td>45</td>
+<td>14.61%</td>
+</tr>
+
+<tr>
+<td>Not yet - but we plan to deploy Kubernetes in our organization soon</td>
+<td>42</td>
+<td>13.64%</td>
+</tr>
+
+<tr>
+<td>Other</td>
+<td>7</td>
+<td>2.27%</td>
+</tr>
+</tbody>
+</table>
+
+<p>The most interesting thing is that there’s nearly 30% of users who do not use Kubernetes,
+and they are not going to move. This means we should keep other deployment options in
+mind when working on Airflow 2.0. On the other hand, almost 70% of the users already
+use Kubernetes, or it’s a viable option for them.</p>
+
+<p><strong>Do you combine multiple DAGs?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>No, I don&rsquo;t combine multiple DAGs</td>
+<td>127</td>
+<td>41.23%</td>
+</tr>
+
+<tr>
+<td>Yes, through SubDAG</td>
+<td>73</td>
+<td>23.70%</td>
+</tr>
+
+<tr>
+<td>Yes, by triggering another DAG</td>
+<td>72</td>
+<td>23.38%</td>
+</tr>
+
+<tr>
+<td>Other</td>
+<td>36</td>
+<td>11.69%</td>
+</tr>
+</tbody>
+</table>
+
+<p>In the other category, 9 people explicitly mentioned using <code>ExternalTaskSensor</code>,
+and I think it could be treated as running subDAGs by triggering other DAGs.</p>
+
+<p><strong>Do you use Airflow Plugins? If yes, what do you use it for?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Adding new operators/sensors and hooks</td>
+<td>187</td>
+<td>60.71%</td>
+</tr>
+
+<tr>
+<td>I don&rsquo;t use Airflow plugins</td>
+<td>109</td>
+<td>35.39%</td>
+</tr>
+
+<tr>
+<td>Adding AppBuilder views &amp; menu items</td>
+<td>31</td>
+<td>10.06%</td>
+</tr>
+
+<tr>
+<td>Adding new executor</td>
+<td>18</td>
+<td>5.84%</td>
+</tr>
+
+<tr>
+<td>Adding OperatorExtraLinks</td>
+<td>7</td>
+<td>2.27%</td>
+</tr>
+</tbody>
+</table>
+
+<p>The high percentage - 60% for “Adding new operators/sensors and hooks” is quite a
+surprising result for some of us - especially that you do not actually need to use the
+plugin mechanism to add any of those. Those are standard python objects, and you can
+simply drop your hooks/operators/sensors code to <code>PYTHONPATH</code> environment variable and
+they will work. It seems that this may be a result of a lack of best practices guide.</p>
+
+<p>Plugins are more useful for adding views and menu items - yet only 10%.
+OperatorExtraLinks are even more useful (though relatively new) feature, so it’s not
+entirely surprising they are hardly used.</p>
+
+<p>It was also kind of surprising that someone at all uses plugins to use their own
+executors. We considered removing that option recently - but now we have to rethink
+our approach.</p>
+
+<p><strong>What metrics do you use to monitor Airflow?</strong></p>
+
+<p>There were a lot of different responses. Some use Prometheus and other services,
+others do not use any monitoring. One of the interesting responses linked to this
+solution for <a href="https://github.com/mastak/airflow_operators_metrics" target="_blank">airflow_operators_metrics</a>.</p>
+
+<h2 id="external-services">External services</h2>
+
+<p><strong>What external services do you use in your Airflow DAGs?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Amazon Web Services</td>
+<td>160</td>
+<td>51.95%</td>
+</tr>
+
+<tr>
+<td>Internal company systems</td>
+<td>150</td>
+<td>48.7%</td>
+</tr>
+
+<tr>
+<td>Hadoop / Spark / Flink / Other Apache software</td>
+<td>119</td>
+<td>38.64%</td>
+</tr>
+
+<tr>
+<td>Google Cloud Platform / Google APIs</td>
+<td>112</td>
+<td>36.36%</td>
+</tr>
+
+<tr>
+<td>Microsoft Azure</td>
+<td>28</td>
+<td>9.09%</td>
+</tr>
+
+<tr>
+<td>I do not use external services in my Airflow DAGs</td>
+<td>18</td>
+<td>5.84%</td>
+</tr>
+</tbody>
+</table>
+
+<p>It’s not surprising that Amazon Web Services is leading the way as they are considered the most mature
+cloud provider. Internal system and other Apache products on the next two positions are
+quite understandable if we take into account that the majority uses Airflow for ETL processes.</p>
+
+<p><strong>What external services do you use in your Airflow DAGs? (Mixed providers)</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Google Cloud Platform / Google APIs, Amazon Web Services</td>
+<td>44</td>
+<td>14.29%</td>
+</tr>
+
+<tr>
+<td>Amazon Web Services, Microsoft Azure</td>
+<td>5</td>
+<td>1.62%</td>
+</tr>
+
+<tr>
+<td>Google Cloud Platform / Google APIs, Microsoft Azure</td>
+<td>4</td>
+<td>1.3%</td>
+</tr>
+</tbody>
+</table>
+
+<p>This result is not surprising because companies usually prefer to stick with one cloud
+provider.</p>
+
+<p><strong>How do you integrate with external services?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Using Bash / Python operator</td>
+<td>220</td>
+<td>71.43%</td>
+</tr>
+
+<tr>
+<td>Using existing, dedicated operators / hooks</td>
+<td>217</td>
+<td>70.45%</td>
+</tr>
+
+<tr>
+<td>Using own, custom operators / hooks</td>
+<td>216</td>
+<td>70.13%</td>
+</tr>
+</tbody>
+</table>
+
+<p>We had some anecdotal evidence that people use more Python/Bash operators than the
+dedicated ones - but it looks like all ways of using Airflow to connect to external
+services are equally popular.</p>
+
+<h2 id="what-can-be-improved">What can be improved</h2>
+
+<p><strong>In your opinion, what could be improved in Airflow?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Scheduler performance</td>
+<td>189</td>
+<td>61.36%</td>
+</tr>
+
+<tr>
+<td>Web UI</td>
+<td>180</td>
+<td>58.44%</td>
+</tr>
+
+<tr>
+<td>Logging, monitoring and alerting</td>
+<td>145</td>
+<td>47.08%</td>
+</tr>
+
+<tr>
+<td>Examples, how-to, onboarding documentation</td>
+<td>143</td>
+<td>46.43%</td>
+</tr>
+
+<tr>
+<td>Technical documentation</td>
+<td>137</td>
+<td>44.48%</td>
+</tr>
+
+<tr>
+<td>Reliability</td>
+<td>112</td>
+<td>36.36%</td>
+</tr>
+
+<tr>
+<td>REST API</td>
+<td>96</td>
+<td>31.17%</td>
+</tr>
+
+<tr>
+<td>Authentication and authorization</td>
+<td>89</td>
+<td>28.9%</td>
+</tr>
+
+<tr>
+<td>External integration e.g. AWS, GCP, Apache product</td>
+<td>49</td>
+<td>15.91%</td>
+</tr>
+
+<tr>
+<td>CLI</td>
+<td>41</td>
+<td>13.31%</td>
+</tr>
+
+<tr>
+<td>I don’t know</td>
+<td>5</td>
+<td>1.62%</td>
+</tr>
+</tbody>
+</table>
+
+<p>The results are rather quite self-explaining. Improved performance of Airflow, better
+UI, and more telemetry are desirable. But this should go in pair with improved
+documentation and resources about using the Airflow, especially when we
+take into account the problem of onboarding new users.</p>
+
+<p>Another interesting point from that question is that only 16% think that operators
+should be extended and improved. This suggests that we should focus on improving
+Airflow core instead of adding more and more integrations.</p>
+
+<p><strong>What would be the most interesting feature for you?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Production-ready Airflow docker image</td>
+<td>175</td>
+<td>56.82%</td>
+</tr>
+
+<tr>
+<td>Declarative way of writing DAGs / automated DAGs generation</td>
+<td>155</td>
+<td>50.32%</td>
+</tr>
+
+<tr>
+<td>Horizontal Autoscaling</td>
+<td>122</td>
+<td>39.61%</td>
+</tr>
+
+<tr>
+<td>Asynchronous Operators</td>
+<td>97</td>
+<td>31.49%</td>
+</tr>
+
+<tr>
+<td>Stateless web server</td>
+<td>81</td>
+<td>26.3%</td>
+</tr>
+
+<tr>
+<td>Knative Executor</td>
+<td>48</td>
+<td>15.58%</td>
+</tr>
+
+<tr>
+<td>I already have all I need</td>
+<td>13</td>
+<td>4.22%</td>
+</tr>
+</tbody>
+</table>
+
+<p>Production Docker image wins, and it’s not a surprise. We all know that deploying
+Airflow is not a plug and play process, and that’s why the official image is being
+worked on by Jarek Potiuk. An unexpected result is that half of the users would like to
+have a declarative way of creating DAGs. That seems to be something that is “against Airflow”
+as we always emphasize the possibility of writing workflows in pure python. Stories
+about DAG generators are not new and confirm that there’s a need for a way to
+declare DAGs.</p>
+
+<h2 id="data">Data</h2>
+
+<p>If you think I missed something and you want to look for insights on your own the data is available
+for you here:</p>
+
+<ul>
+<li>Original data: <a href="https://storage.googleapis.com/airflow-survey/survey.csv" target="_blank">https://storage.googleapis.com/airflow-survey/survey.csv</a></li>
+<li>Processed: <a href="https://storage.googleapis.com/airflow-survey/airflow_survey_processed.csv" target="_blank">https://storage.googleapis.com/airflow-survey/airflow_survey_processed.csv</a></li>
+</ul>
+
+<p>The processed data includes multi-choice options one-hot encoded. If you find any interesting
+insight, please update the article (<a href="https://github.com/apache/airflow-site/blob/aip-11/CONTRIBUTE.md" target="_blank">make PR</a>
+to Airflow site).</p>
+
+ </description>
+ </item>
+
+ <item>
+ <title>Blog: New Airflow website</title>
+ <link>/blog/announcing-new-website/</link>
+ <pubDate>Wed, 11 Dec 2019 00:00:00 +0000</pubDate>
+
+ <guid>/blog/announcing-new-website/</guid>
+ <description>
+
+
+ <p>The brand <a href="https://airflow.apache.org/" target="_blank">new Airflow website</a> has arrived! Those who have been following the process know that the journey to update <a href="https://airflow.readthedocs.io/en/1.10.6/" target="_blank">the old Airflow website</a> started at the beginning of the year.
+Thanks to sponsorship from the Cloud Composer team at Google that allowed us to
+collaborate with <a href="https://www.polidea.com/" target="_blank">Polidea</a> and with their design studio <a href="https://utilodesign.com/" target="_blank">Utilo</a>, and deliver an awesome website.</p>
+
+<p>Documentation of open source projects is key to engaging new contributors in the maintenance,
+development, and adoption of software. We want the Apache Airflow community to have
+the best possible experience to contribute and use the project. We also took this opportunity to make the project
+more accessible, and in doing so, increase its reach.</p>
+
+<p>In the past three and a half months, we have updated everything: created a more efficient landing page,
+enhanced information architecture, and improved UX &amp; UI. Most importantly, the website now has capabilities
+to be translated into many languages. This is our effort to foster a more inclusive community around
+Apache Airflow, and we look forward to seeing contributions in Spanish, Chinese, Russian, and other languages as well!</p>
+
+<p>We built our website on Docsy, a platform that is easy to use and contribute to. Follow
+<a href="https://github.com/apache/airflow-site/blob/aip-11/README.md" target="_blank">these steps</a> to set up your environment and
+to create your first pull request. You may also use
+the new website for your own open source project as a template.
+All of our <a href="https://github.com/apache/airflow-site/tree/aip-11" target="_blank">code is open and hosted on Github</a>.</p>
+
+<p>Share your questions, comments, and suggestions with us, to help us improve the website.
+We hope that this new design makes finding documentation about Airflow easier,
+and that its improved accessibility increases adoption and use of Apache Airflow around the world.</p>
+
+<p>Happy browsing!</p>
+
+ </description>
+ </item>
+
+ <item>
<title>Blog: ApacheCon Europe 2019 — Thoughts and Insights by Airflow Commiters</title>
<link>/blog/apache-con-europe-2019-thoughts-and-insights-by-airflow-commiters/</link>
<pubDate>Fri, 22 Nov 2019 00:00:00 +0000</pubDate>
diff --git a/blog/tags/development/index.html b/blog/tags/development/index.html
index ea51f1f..bf8bc82 100644
--- a/blog/tags/development/index.html
+++ b/blog/tags/development/index.html
@@ -78,15 +78,15 @@ if (!doNotTrack) {
- <link rel="preload" href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" as="style">
- <link href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" rel="stylesheet" integrity="">
+ <link rel="preload" href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" as="style">
+ <link href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" rel="stylesheet" integrity="">
-<link rel="preload" href="/main.b2992.js" as="script">
-<link rel="preload" href="/chunk-1.b2992.js" as="script">
+<link rel="preload" href="/main.e3a02.js" as="script">
+<link rel="preload" href="/chunk-1.e3a02.js" as="script">
-<link rel="preload" href="/chunk-4.b2992.js" as="script">
+<link rel="preload" href="/chunk-4.e3a02.js" as="script">
@@ -395,6 +395,16 @@ if (!doNotTrack) {
href="/blog/tags/development/">Development</a>
+
+ <a class="tag "
+ href="/blog/tags/survey/">Survey</a>
+
+
+
+ <a class="tag "
+ href="/blog/tags/users/">Users</a>
+
+
</div>
</div>
<div class="blog__list-items-wrapper">
@@ -621,6 +631,6 @@ if (!doNotTrack) {
</body>
- <script src="/main.b2992.js"></script>
+ <script src="/main.e3a02.js"></script>
</html>
diff --git a/blog/tags/community/index.html b/blog/tags/survey/index.html
similarity index 96%
copy from blog/tags/community/index.html
copy to blog/tags/survey/index.html
index fb899ac..0e9cbdb 100644
--- a/blog/tags/community/index.html
+++ b/blog/tags/survey/index.html
@@ -9,7 +9,7 @@
<META NAME="ROBOTS" CONTENT="INDEX, FOLLOW">
-<link rel="alternate" type="application/rss+xml" href="/blog/tags/community/index.xml">
+<link rel="alternate" type="application/rss+xml" href="/blog/tags/survey/index.xml">
<link rel="apple-touch-icon" sizes="57x57" href="/favicons/apple-icon-57x57.png">
@@ -30,21 +30,21 @@
<meta name="msapplication-TileImage" content="/favicons/ms-icon-144x144.png">
<meta name="theme-color" content="#ffffff">
-<title>Community | Apache Airflow</title><meta property="og:title" content="Community" />
+<title>survey | Apache Airflow</title><meta property="og:title" content="survey" />
<meta property="og:description" content="Platform created by community to programmatically author, schedule and monitor workflows." />
<meta property="og:type" content="website" />
-<meta property="og:url" content="/blog/tags/community/" />
+<meta property="og:url" content="/blog/tags/survey/" />
<meta property="og:image" content="/images/feature-image.png" />
-<meta property="og:updated_time" content="2019-11-22T00:00:00+00:00" /><meta property="og:site_name" content="Apache Airflow" />
-<meta itemprop="name" content="Community">
+<meta property="og:updated_time" content="2019-12-11T00:00:00+00:00" /><meta property="og:site_name" content="Apache Airflow" />
+<meta itemprop="name" content="survey">
<meta itemprop="description" content="Platform created by community to programmatically author, schedule and monitor workflows.">
<meta name="twitter:card" content="summary_large_image"/>
<meta name="twitter:image" content="/images/feature-image.png"/>
-<meta name="twitter:title" content="Community"/>
+<meta name="twitter:title" content="survey"/>
<meta name="twitter:description" content="Platform created by community to programmatically author, schedule and monitor workflows."/>
@@ -78,15 +78,15 @@ if (!doNotTrack) {
- <link rel="preload" href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" as="style">
- <link href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" rel="stylesheet" integrity="">
+ <link rel="preload" href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" as="style">
+ <link href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" rel="stylesheet" integrity="">
-<link rel="preload" href="/main.b2992.js" as="script">
-<link rel="preload" href="/chunk-1.b2992.js" as="script">
+<link rel="preload" href="/main.e3a02.js" as="script">
+<link rel="preload" href="/chunk-1.e3a02.js" as="script">
-<link rel="preload" href="/chunk-4.b2992.js" as="script">
+<link rel="preload" href="/chunk-4.e3a02.js" as="script">
@@ -386,7 +386,7 @@ if (!doNotTrack) {
<a class="tag" href="/blog/">All</a>
- <a class="tag active"
+ <a class="tag "
href="/blog/tags/community/">Community</a>
@@ -395,6 +395,16 @@ if (!doNotTrack) {
href="/blog/tags/development/">Development</a>
+
+ <a class="tag active"
+ href="/blog/tags/survey/">Survey</a>
+
+
+
+ <a class="tag "
+ href="/blog/tags/users/">Users</a>
+
+
</div>
</div>
<div class="blog__list-items-wrapper">
@@ -415,14 +425,22 @@ if (!doNotTrack) {
<a class="tag" href="/blog/tags/community/">Community</a>
+
+ <a class="tag" href="/blog/tags/survey/">Survey</a>
+
+
+
+ <a class="tag" href="/blog/tags/users/">Users</a>
+
+
</div>
- <span class="bodytext__medium--brownish-grey box-event__blogpost--date">Fri, Nov 22, 2019</span>
+ <span class="bodytext__medium--brownish-grey box-event__blogpost--date">Wed, Dec 11, 2019</span>
</div>
- <p class="box-event__blogpost--header">ApacheCon Europe 2019 — Thoughts and Insights by Airflow Commiters</p>
- <p class="box-event__blogpost--author">Polidea</p>
- <p class="box-event__blogpost--description">Here come some thoughts by Airflow committers and contributors from the ApacheCon Europe 2019. Get to know the ASF community!</p>
+ <p class="box-event__blogpost--header">Airflow Survey 2019</p>
+ <p class="box-event__blogpost--author">Tomek Urbaszek</p>
+ <p class="box-event__blogpost--description">Receiving and adjusting to our users’ feedback is a must. Let’s see who Airflow users are, how they play with it, and what they miss.</p>
<div class="mt-auto">
- <a href="/blog/apache-con-europe-2019-thoughts-and-insights-by-airflow-commiters/">
+ <a href="/blog/airflow-survey/">
<button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" >Read more</button>
@@ -443,7 +461,7 @@ if (!doNotTrack) {
<div class="base-layout--button">
- <a href=https://github.com/apache/airflow-site/edit/aip-11/landing-pages/site/content/en/tags/community>
+ <a href=https://github.com/apache/airflow-site/edit/aip-11/landing-pages/site/content/en/tags/survey>
@@ -621,6 +639,6 @@ if (!doNotTrack) {
</body>
- <script src="/main.b2992.js"></script>
+ <script src="/main.e3a02.js"></script>
</html>
diff --git a/blog/tags/survey/index.xml b/blog/tags/survey/index.xml
new file mode 100644
index 0000000..5630b3e
--- /dev/null
+++ b/blog/tags/survey/index.xml
@@ -0,0 +1,1006 @@
+<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">
+ <channel>
+ <title>Apache Airflow – survey</title>
+ <link>/blog/tags/survey/</link>
+ <description>Recent content in survey on Apache Airflow</description>
+ <generator>Hugo -- gohugo.io</generator>
+ <lastBuildDate>Wed, 11 Dec 2019 00:00:00 +0000</lastBuildDate>
+
+ <atom:link href="/blog/tags/survey/index.xml" rel="self" type="application/rss+xml" />
+
+
+
+
+
+
+
+ <item>
+ <title>Blog: Airflow Survey 2019</title>
+ <link>/blog/airflow-survey/</link>
+ <pubDate>Wed, 11 Dec 2019 00:00:00 +0000</pubDate>
+
+ <guid>/blog/airflow-survey/</guid>
+ <description>
+
+
+
+
+<h1 id="apache-airflow-survey-2019">Apache Airflow Survey 2019</h1>
+
+<p>Apache Airflow is <a href="https://www.astronomer.io/blog/why-airflow/" target="_blank">growing faster than ever</a>.
+Thus, receiving and adjusting to our users’ feedback is a must. We created
+<a href="https://forms.gle/XAzR1pQBZiftvPQM7" target="_blank">survey</a> and we got <strong>308</strong> responses.
+Let’s see who Airflow users are, how they play with it, and what they miss.</p>
+
+<h1 id="overview-of-the-user">Overview of the user</h1>
+
+<p><strong>What best describes your current occupation?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Data Engineer</td>
+<td>194</td>
+<td>62.99%</td>
+</tr>
+
+<tr>
+<td>Developer</td>
+<td>34</td>
+<td>11.04%</td>
+</tr>
+
+<tr>
+<td>Architect</td>
+<td>23</td>
+<td>7.47%</td>
+</tr>
+
+<tr>
+<td>Data Scientist</td>
+<td>19</td>
+<td>6.17%</td>
+</tr>
+
+<tr>
+<td>Data Analyst</td>
+<td>13</td>
+<td>4.22%</td>
+</tr>
+
+<tr>
+<td>DevOps</td>
+<td>13</td>
+<td>4.22%</td>
+</tr>
+
+<tr>
+<td>IT Administrator</td>
+<td>2</td>
+<td>0.65%</td>
+</tr>
+
+<tr>
+<td>Machine Learning Engineer</td>
+<td>2</td>
+<td>0.65%</td>
+</tr>
+
+<tr>
+<td>Manager</td>
+<td>2</td>
+<td>0.65%</td>
+</tr>
+
+<tr>
+<td>Operations</td>
+<td>2</td>
+<td>0.65%</td>
+</tr>
+
+<tr>
+<td>Chief Data Officer</td>
+<td>1</td>
+<td>0.32%</td>
+</tr>
+
+<tr>
+<td>Engineering Manager</td>
+<td>1</td>
+<td>0.32%</td>
+</tr>
+
+<tr>
+<td>Intern</td>
+<td>1</td>
+<td>0.32%</td>
+</tr>
+
+<tr>
+<td>Product owner</td>
+<td>1</td>
+<td>0.32%</td>
+</tr>
+
+<tr>
+<td>Quant</td>
+<td>1</td>
+<td>0.32%</td>
+</tr>
+</tbody>
+</table>
+
+<p><strong>In your day to day job, what do you use Airflow for?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Data processing (ETL)</td>
+<td>298</td>
+<td>96.75%</td>
+</tr>
+
+<tr>
+<td>Artificial Intelligence and Machine Learning Pipelines</td>
+<td>90</td>
+<td>29.22%</td>
+</tr>
+
+<tr>
+<td>Automating DevOps operations</td>
+<td>64</td>
+<td>20.78%</td>
+</tr>
+</tbody>
+</table>
+
+<p>According to the survey, most of the Airflow users are the “data” people. Moreover,
+28.57% uses Airflow to both ETL and ML pipelines meaning that those two fields
+are somehow connected. Only five respondents use Airflow for DevOps operations only,
+That means that other 59 people who use Airflow for DevOps stuff use it also for
+ETL / ML purposes.</p>
+
+<p><strong>How many active DAGs do you have in your largest Airflow instance?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>0-20</td>
+<td>115</td>
+<td>37.34%</td>
+</tr>
+
+<tr>
+<td>21-40</td>
+<td>65</td>
+<td>21.10%</td>
+</tr>
+
+<tr>
+<td>41-60</td>
+<td>44</td>
+<td>14.29%</td>
+</tr>
+
+<tr>
+<td>61-100</td>
+<td>28</td>
+<td>9.09%</td>
+</tr>
+
+<tr>
+<td>101-200</td>
+<td>28</td>
+<td>9.09%</td>
+</tr>
+
+<tr>
+<td>201-300</td>
+<td>7</td>
+<td>2.27%</td>
+</tr>
+
+<tr>
+<td>301-999</td>
+<td>8</td>
+<td>2.60%</td>
+</tr>
+
+<tr>
+<td>1000+</td>
+<td>13</td>
+<td>4.22%</td>
+</tr>
+</tbody>
+</table>
+
+<p>The majority of users do not exceed 100 active DAGs per Airflow instance. However,
+as we can see there are users who exceed thousands of DAGs with a maximum number 5000.</p>
+
+<p><strong>What is the maximum number of tasks that you have used in one DAG?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>0-10</td>
+<td>61</td>
+<td>19.81%</td>
+</tr>
+
+<tr>
+<td>11-20</td>
+<td>60</td>
+<td>19.48%</td>
+</tr>
+
+<tr>
+<td>21-30</td>
+<td>31</td>
+<td>10.06%</td>
+</tr>
+
+<tr>
+<td>31-40</td>
+<td>21</td>
+<td>6.82%</td>
+</tr>
+
+<tr>
+<td>41-50</td>
+<td>26</td>
+<td>8.44%</td>
+</tr>
+
+<tr>
+<td>51-100</td>
+<td>36</td>
+<td>11.69%</td>
+</tr>
+
+<tr>
+<td>101-200</td>
+<td>28</td>
+<td>9.09%</td>
+</tr>
+
+<tr>
+<td>201-500</td>
+<td>21</td>
+<td>6.82%</td>
+</tr>
+
+<tr>
+<td>501+</td>
+<td>24</td>
+<td>11.54%</td>
+</tr>
+</tbody>
+</table>
+
+<p>The given maximum number of tasks in a single DAG was 10 000 (!). The number of tasks
+depends on the purposes of a DAG, so it’s rather hard to say if users have “simple”
+or “complicated” workflows.</p>
+
+<p><strong>When onboarding new members to Airflow, what is the biggest problem?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>No guide on best practises on developing DAGs</td>
+<td>160</td>
+<td>51.95%</td>
+</tr>
+
+<tr>
+<td>Small number of tutorials on different aspects of using Airflow</td>
+<td>57</td>
+<td>18.51%</td>
+</tr>
+
+<tr>
+<td>Documentation is not clear enough</td>
+<td>42</td>
+<td>13.64%</td>
+</tr>
+
+<tr>
+<td>Small number of blogs regarding Airflow</td>
+<td>6</td>
+<td>1.95%</td>
+</tr>
+
+<tr>
+<td>Other</td>
+<td>43</td>
+<td>13.96%</td>
+</tr>
+</tbody>
+</table>
+
+<p>This is an important result. Using Airflow is all about writing and scheduling DAGs.
+No guide or any other complete resource on best practices for developing Dags is a big
+problem. Diving deep in the “other” answers, we can find that:</p>
+
+<ul>
+<li>Airflow’s “magic” (scheduler, executors, schedule times) is hard to understand</li>
+<li>DAG testing is not easy to do and to explain</li>
+<li>Airflow UI needs some love.</li>
+</ul>
+
+<p><strong>How likely are you to recommend Apache Airflow?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Very Likely</td>
+<td>140</td>
+<td>45.45%</td>
+</tr>
+
+<tr>
+<td>Likely</td>
+<td>124</td>
+<td>40.26%</td>
+</tr>
+
+<tr>
+<td>Neutral</td>
+<td>33</td>
+<td>10.71%</td>
+</tr>
+
+<tr>
+<td>Unlikely</td>
+<td>8</td>
+<td>2.60%</td>
+</tr>
+
+<tr>
+<td>Very unlikely</td>
+<td>3</td>
+<td>0.97%</td>
+</tr>
+</tbody>
+</table>
+
+<p>This means that more than 85% of people who use Airflow like it. It seems Airflow does
+its job nicely. However, we have to remember that this survey is likely biased - it’s
+more likely that you respond to the survey if you like the tool you use. Should we
+focus then on those 11 people who did not like Airflow? It’s a good question.</p>
+
+<h2 id="airflow-usage">Airflow usage</h2>
+
+<p><strong>Which interface(s) of Airflow do you use as part of your current role?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Original Airflow Graphical User Interface</td>
+<td>297</td>
+<td>96.43%</td>
+</tr>
+
+<tr>
+<td>CLI</td>
+<td>126</td>
+<td>40.91%</td>
+</tr>
+
+<tr>
+<td>Original Airflow Graphical User Interface, CLI</td>
+<td>117</td>
+<td>37.99%</td>
+</tr>
+
+<tr>
+<td>API</td>
+<td>60</td>
+<td>19.48%</td>
+</tr>
+
+<tr>
+<td>Original Airflow Graphical User Interface, CLI, API</td>
+<td>32</td>
+<td>10.39%</td>
+</tr>
+
+<tr>
+<td>Custom (own created) Airflow Graphical User Interface</td>
+<td>25</td>
+<td>8.12%</td>
+</tr>
+</tbody>
+</table>
+
+<p>It’s visible that usage of CLI goes in pair with using Airflow web UI. Our
+survey included some UX related questions to allow us to understand how users
+use Airflow webserver.</p>
+
+<p><strong>What do you use the Graphical User Interface for?</strong></p>
+
+<p><img src="plot1.png" alt="" /></p>
+
+<p><strong>What do you use CLI for?</strong></p>
+
+<p><img src="plot2.png" alt="" /></p>
+
+<p><strong>In Airflow, which UI view(s) are important for you?</strong></p>
+
+<p><img src="plot3.png" alt="" /></p>
+
+<p>Here we see that the majority uses Web UI mostly for monitoring purposes:</p>
+
+<ul>
+<li>Monitoring DAGs</li>
+<li>Accessing logs</li>
+</ul>
+
+<p>An interesting result is that many people seem not to use backfilling as
+there’s no other way than to do it by CLI.</p>
+
+<p><strong>What executor type do you use?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Celery</td>
+<td>138</td>
+<td>44.81%</td>
+</tr>
+
+<tr>
+<td>Local</td>
+<td>85</td>
+<td>27.60%</td>
+</tr>
+
+<tr>
+<td>Kubernetes</td>
+<td>52</td>
+<td>16.88%</td>
+</tr>
+
+<tr>
+<td>Sequential</td>
+<td>22</td>
+<td>7.14%</td>
+</tr>
+
+<tr>
+<td>Other</td>
+<td>11</td>
+<td>3.57</td>
+</tr>
+</tbody>
+</table>
+
+<p>The other option mostly consisted of information that someone uses a few types or is
+migrating from one executor to another. What can be observed is an increase in usage
+of Local and Kubernetes executors when compared to results from an earlier <a href="https://ash.berlintaylor.com/writings/2019/02/airflow-user-survey-2019/" target="_blank">survey done
+by Ash</a>.</p>
+
+<p><strong>Do you use Kubernetes-based deployments for Airflow?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>No - we do not plan to use Kubernetes near term</td>
+<td>88</td>
+<td>28.57%</td>
+</tr>
+
+<tr>
+<td>Yes - setup on our own via Helm Chart or similar</td>
+<td>65</td>
+<td>21.10%</td>
+</tr>
+
+<tr>
+<td>Not yet - but we use Kubernetes in our organization and we could move</td>
+<td>61</td>
+<td>19.81%</td>
+</tr>
+
+<tr>
+<td>Yes - via managed service in the cloud (Composer / Astronomer etc.)</td>
+<td>45</td>
+<td>14.61%</td>
+</tr>
+
+<tr>
+<td>Not yet - but we plan to deploy Kubernetes in our organization soon</td>
+<td>42</td>
+<td>13.64%</td>
+</tr>
+
+<tr>
+<td>Other</td>
+<td>7</td>
+<td>2.27%</td>
+</tr>
+</tbody>
+</table>
+
+<p>The most interesting thing is that there’s nearly 30% of users who do not use Kubernetes,
+and they are not going to move. This means we should keep other deployment options in
+mind when working on Airflow 2.0. On the other hand, almost 70% of the users already
+use Kubernetes, or it’s a viable option for them.</p>
+
+<p><strong>Do you combine multiple DAGs?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>No, I don&rsquo;t combine multiple DAGs</td>
+<td>127</td>
+<td>41.23%</td>
+</tr>
+
+<tr>
+<td>Yes, through SubDAG</td>
+<td>73</td>
+<td>23.70%</td>
+</tr>
+
+<tr>
+<td>Yes, by triggering another DAG</td>
+<td>72</td>
+<td>23.38%</td>
+</tr>
+
+<tr>
+<td>Other</td>
+<td>36</td>
+<td>11.69%</td>
+</tr>
+</tbody>
+</table>
+
+<p>In the other category, 9 people explicitly mentioned using <code>ExternalTaskSensor</code>,
+and I think it could be treated as running subDAGs by triggering other DAGs.</p>
+
+<p><strong>Do you use Airflow Plugins? If yes, what do you use it for?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Adding new operators/sensors and hooks</td>
+<td>187</td>
+<td>60.71%</td>
+</tr>
+
+<tr>
+<td>I don&rsquo;t use Airflow plugins</td>
+<td>109</td>
+<td>35.39%</td>
+</tr>
+
+<tr>
+<td>Adding AppBuilder views &amp; menu items</td>
+<td>31</td>
+<td>10.06%</td>
+</tr>
+
+<tr>
+<td>Adding new executor</td>
+<td>18</td>
+<td>5.84%</td>
+</tr>
+
+<tr>
+<td>Adding OperatorExtraLinks</td>
+<td>7</td>
+<td>2.27%</td>
+</tr>
+</tbody>
+</table>
+
+<p>The high percentage - 60% for “Adding new operators/sensors and hooks” is quite a
+surprising result for some of us - especially that you do not actually need to use the
+plugin mechanism to add any of those. Those are standard python objects, and you can
+simply drop your hooks/operators/sensors code to <code>PYTHONPATH</code> environment variable and
+they will work. It seems that this may be a result of a lack of best practices guide.</p>
+
+<p>Plugins are more useful for adding views and menu items - yet only 10%.
+OperatorExtraLinks are even more useful (though relatively new) feature, so it’s not
+entirely surprising they are hardly used.</p>
+
+<p>It was also kind of surprising that someone at all uses plugins to use their own
+executors. We considered removing that option recently - but now we have to rethink
+our approach.</p>
+
+<p><strong>What metrics do you use to monitor Airflow?</strong></p>
+
+<p>There were a lot of different responses. Some use Prometheus and other services,
+others do not use any monitoring. One of the interesting responses linked to this
+solution for <a href="https://github.com/mastak/airflow_operators_metrics" target="_blank">airflow_operators_metrics</a>.</p>
+
+<h2 id="external-services">External services</h2>
+
+<p><strong>What external services do you use in your Airflow DAGs?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Amazon Web Services</td>
+<td>160</td>
+<td>51.95%</td>
+</tr>
+
+<tr>
+<td>Internal company systems</td>
+<td>150</td>
+<td>48.7%</td>
+</tr>
+
+<tr>
+<td>Hadoop / Spark / Flink / Other Apache software</td>
+<td>119</td>
+<td>38.64%</td>
+</tr>
+
+<tr>
+<td>Google Cloud Platform / Google APIs</td>
+<td>112</td>
+<td>36.36%</td>
+</tr>
+
+<tr>
+<td>Microsoft Azure</td>
+<td>28</td>
+<td>9.09%</td>
+</tr>
+
+<tr>
+<td>I do not use external services in my Airflow DAGs</td>
+<td>18</td>
+<td>5.84%</td>
+</tr>
+</tbody>
+</table>
+
+<p>It’s not surprising that Amazon Web Services is leading the way as they are considered the most mature
+cloud provider. Internal system and other Apache products on the next two positions are
+quite understandable if we take into account that the majority uses Airflow for ETL processes.</p>
+
+<p><strong>What external services do you use in your Airflow DAGs? (Mixed providers)</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Google Cloud Platform / Google APIs, Amazon Web Services</td>
+<td>44</td>
+<td>14.29%</td>
+</tr>
+
+<tr>
+<td>Amazon Web Services, Microsoft Azure</td>
+<td>5</td>
+<td>1.62%</td>
+</tr>
+
+<tr>
+<td>Google Cloud Platform / Google APIs, Microsoft Azure</td>
+<td>4</td>
+<td>1.3%</td>
+</tr>
+</tbody>
+</table>
+
+<p>This result is not surprising because companies usually prefer to stick with one cloud
+provider.</p>
+
+<p><strong>How do you integrate with external services?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Using Bash / Python operator</td>
+<td>220</td>
+<td>71.43%</td>
+</tr>
+
+<tr>
+<td>Using existing, dedicated operators / hooks</td>
+<td>217</td>
+<td>70.45%</td>
+</tr>
+
+<tr>
+<td>Using own, custom operators / hooks</td>
+<td>216</td>
+<td>70.13%</td>
+</tr>
+</tbody>
+</table>
+
+<p>We had some anecdotal evidence that people use more Python/Bash operators than the
+dedicated ones - but it looks like all ways of using Airflow to connect to external
+services are equally popular.</p>
+
+<h2 id="what-can-be-improved">What can be improved</h2>
+
+<p><strong>In your opinion, what could be improved in Airflow?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Scheduler performance</td>
+<td>189</td>
+<td>61.36%</td>
+</tr>
+
+<tr>
+<td>Web UI</td>
+<td>180</td>
+<td>58.44%</td>
+</tr>
+
+<tr>
+<td>Logging, monitoring and alerting</td>
+<td>145</td>
+<td>47.08%</td>
+</tr>
+
+<tr>
+<td>Examples, how-to, onboarding documentation</td>
+<td>143</td>
+<td>46.43%</td>
+</tr>
+
+<tr>
+<td>Technical documentation</td>
+<td>137</td>
+<td>44.48%</td>
+</tr>
+
+<tr>
+<td>Reliability</td>
+<td>112</td>
+<td>36.36%</td>
+</tr>
+
+<tr>
+<td>REST API</td>
+<td>96</td>
+<td>31.17%</td>
+</tr>
+
+<tr>
+<td>Authentication and authorization</td>
+<td>89</td>
+<td>28.9%</td>
+</tr>
+
+<tr>
+<td>External integration e.g. AWS, GCP, Apache product</td>
+<td>49</td>
+<td>15.91%</td>
+</tr>
+
+<tr>
+<td>CLI</td>
+<td>41</td>
+<td>13.31%</td>
+</tr>
+
+<tr>
+<td>I don’t know</td>
+<td>5</td>
+<td>1.62%</td>
+</tr>
+</tbody>
+</table>
+
+<p>The results are rather quite self-explaining. Improved performance of Airflow, better
+UI, and more telemetry are desirable. But this should go in pair with improved
+documentation and resources about using the Airflow, especially when we
+take into account the problem of onboarding new users.</p>
+
+<p>Another interesting point from that question is that only 16% think that operators
+should be extended and improved. This suggests that we should focus on improving
+Airflow core instead of adding more and more integrations.</p>
+
+<p><strong>What would be the most interesting feature for you?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Production-ready Airflow docker image</td>
+<td>175</td>
+<td>56.82%</td>
+</tr>
+
+<tr>
+<td>Declarative way of writing DAGs / automated DAGs generation</td>
+<td>155</td>
+<td>50.32%</td>
+</tr>
+
+<tr>
+<td>Horizontal Autoscaling</td>
+<td>122</td>
+<td>39.61%</td>
+</tr>
+
+<tr>
+<td>Asynchronous Operators</td>
+<td>97</td>
+<td>31.49%</td>
+</tr>
+
+<tr>
+<td>Stateless web server</td>
+<td>81</td>
+<td>26.3%</td>
+</tr>
+
+<tr>
+<td>Knative Executor</td>
+<td>48</td>
+<td>15.58%</td>
+</tr>
+
+<tr>
+<td>I already have all I need</td>
+<td>13</td>
+<td>4.22%</td>
+</tr>
+</tbody>
+</table>
+
+<p>Production Docker image wins, and it’s not a surprise. We all know that deploying
+Airflow is not a plug and play process, and that’s why the official image is being
+worked on by Jarek Potiuk. An unexpected result is that half of the users would like to
+have a declarative way of creating DAGs. That seems to be something that is “against Airflow”
+as we always emphasize the possibility of writing workflows in pure python. Stories
+about DAG generators are not new and confirm that there’s a need for a way to
+declare DAGs.</p>
+
+<h2 id="data">Data</h2>
+
+<p>If you think I missed something and you want to look for insights on your own the data is available
+for you here:</p>
+
+<ul>
+<li>Original data: <a href="https://storage.googleapis.com/airflow-survey/survey.csv" target="_blank">https://storage.googleapis.com/airflow-survey/survey.csv</a></li>
+<li>Processed: <a href="https://storage.googleapis.com/airflow-survey/airflow_survey_processed.csv" target="_blank">https://storage.googleapis.com/airflow-survey/airflow_survey_processed.csv</a></li>
+</ul>
+
+<p>The processed data includes multi-choice options one-hot encoded. If you find any interesting
+insight, please update the article (<a href="https://github.com/apache/airflow-site/blob/aip-11/CONTRIBUTE.md" target="_blank">make PR</a>
+to Airflow site).</p>
+
+ </description>
+ </item>
+
+ </channel>
+</rss>
diff --git a/blog/tags/community/index.html b/blog/tags/users/index.html
similarity index 96%
copy from blog/tags/community/index.html
copy to blog/tags/users/index.html
index fb899ac..d5f63fd 100644
--- a/blog/tags/community/index.html
+++ b/blog/tags/users/index.html
@@ -9,7 +9,7 @@
<META NAME="ROBOTS" CONTENT="INDEX, FOLLOW">
-<link rel="alternate" type="application/rss+xml" href="/blog/tags/community/index.xml">
+<link rel="alternate" type="application/rss+xml" href="/blog/tags/users/index.xml">
<link rel="apple-touch-icon" sizes="57x57" href="/favicons/apple-icon-57x57.png">
@@ -30,21 +30,21 @@
<meta name="msapplication-TileImage" content="/favicons/ms-icon-144x144.png">
<meta name="theme-color" content="#ffffff">
-<title>Community | Apache Airflow</title><meta property="og:title" content="Community" />
+<title>users | Apache Airflow</title><meta property="og:title" content="users" />
<meta property="og:description" content="Platform created by community to programmatically author, schedule and monitor workflows." />
<meta property="og:type" content="website" />
-<meta property="og:url" content="/blog/tags/community/" />
+<meta property="og:url" content="/blog/tags/users/" />
<meta property="og:image" content="/images/feature-image.png" />
-<meta property="og:updated_time" content="2019-11-22T00:00:00+00:00" /><meta property="og:site_name" content="Apache Airflow" />
-<meta itemprop="name" content="Community">
+<meta property="og:updated_time" content="2019-12-11T00:00:00+00:00" /><meta property="og:site_name" content="Apache Airflow" />
+<meta itemprop="name" content="users">
<meta itemprop="description" content="Platform created by community to programmatically author, schedule and monitor workflows.">
<meta name="twitter:card" content="summary_large_image"/>
<meta name="twitter:image" content="/images/feature-image.png"/>
-<meta name="twitter:title" content="Community"/>
+<meta name="twitter:title" content="users"/>
<meta name="twitter:description" content="Platform created by community to programmatically author, schedule and monitor workflows."/>
@@ -78,15 +78,15 @@ if (!doNotTrack) {
- <link rel="preload" href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" as="style">
- <link href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" rel="stylesheet" integrity="">
+ <link rel="preload" href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" as="style">
+ <link href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" rel="stylesheet" integrity="">
-<link rel="preload" href="/main.b2992.js" as="script">
-<link rel="preload" href="/chunk-1.b2992.js" as="script">
+<link rel="preload" href="/main.e3a02.js" as="script">
+<link rel="preload" href="/chunk-1.e3a02.js" as="script">
-<link rel="preload" href="/chunk-4.b2992.js" as="script">
+<link rel="preload" href="/chunk-4.e3a02.js" as="script">
@@ -386,7 +386,7 @@ if (!doNotTrack) {
<a class="tag" href="/blog/">All</a>
- <a class="tag active"
+ <a class="tag "
href="/blog/tags/community/">Community</a>
@@ -395,6 +395,16 @@ if (!doNotTrack) {
href="/blog/tags/development/">Development</a>
+
+ <a class="tag "
+ href="/blog/tags/survey/">Survey</a>
+
+
+
+ <a class="tag active"
+ href="/blog/tags/users/">Users</a>
+
+
</div>
</div>
<div class="blog__list-items-wrapper">
@@ -415,14 +425,22 @@ if (!doNotTrack) {
<a class="tag" href="/blog/tags/community/">Community</a>
+
+ <a class="tag" href="/blog/tags/survey/">Survey</a>
+
+
+
+ <a class="tag" href="/blog/tags/users/">Users</a>
+
+
</div>
- <span class="bodytext__medium--brownish-grey box-event__blogpost--date">Fri, Nov 22, 2019</span>
+ <span class="bodytext__medium--brownish-grey box-event__blogpost--date">Wed, Dec 11, 2019</span>
</div>
- <p class="box-event__blogpost--header">ApacheCon Europe 2019 — Thoughts and Insights by Airflow Commiters</p>
- <p class="box-event__blogpost--author">Polidea</p>
- <p class="box-event__blogpost--description">Here come some thoughts by Airflow committers and contributors from the ApacheCon Europe 2019. Get to know the ASF community!</p>
+ <p class="box-event__blogpost--header">Airflow Survey 2019</p>
+ <p class="box-event__blogpost--author">Tomek Urbaszek</p>
+ <p class="box-event__blogpost--description">Receiving and adjusting to our users’ feedback is a must. Let’s see who Airflow users are, how they play with it, and what they miss.</p>
<div class="mt-auto">
- <a href="/blog/apache-con-europe-2019-thoughts-and-insights-by-airflow-commiters/">
+ <a href="/blog/airflow-survey/">
<button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" >Read more</button>
@@ -443,7 +461,7 @@ if (!doNotTrack) {
<div class="base-layout--button">
- <a href=https://github.com/apache/airflow-site/edit/aip-11/landing-pages/site/content/en/tags/community>
+ <a href=https://github.com/apache/airflow-site/edit/aip-11/landing-pages/site/content/en/tags/users>
@@ -621,6 +639,6 @@ if (!doNotTrack) {
</body>
- <script src="/main.b2992.js"></script>
+ <script src="/main.e3a02.js"></script>
</html>
diff --git a/blog/tags/users/index.xml b/blog/tags/users/index.xml
new file mode 100644
index 0000000..3a09e82
--- /dev/null
+++ b/blog/tags/users/index.xml
@@ -0,0 +1,1006 @@
+<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">
+ <channel>
+ <title>Apache Airflow – users</title>
+ <link>/blog/tags/users/</link>
+ <description>Recent content in users on Apache Airflow</description>
+ <generator>Hugo -- gohugo.io</generator>
+ <lastBuildDate>Wed, 11 Dec 2019 00:00:00 +0000</lastBuildDate>
+
+ <atom:link href="/blog/tags/users/index.xml" rel="self" type="application/rss+xml" />
+
+
+
+
+
+
+
+ <item>
+ <title>Blog: Airflow Survey 2019</title>
+ <link>/blog/airflow-survey/</link>
+ <pubDate>Wed, 11 Dec 2019 00:00:00 +0000</pubDate>
+
+ <guid>/blog/airflow-survey/</guid>
+ <description>
+
+
+
+
+<h1 id="apache-airflow-survey-2019">Apache Airflow Survey 2019</h1>
+
+<p>Apache Airflow is <a href="https://www.astronomer.io/blog/why-airflow/" target="_blank">growing faster than ever</a>.
+Thus, receiving and adjusting to our users’ feedback is a must. We created
+<a href="https://forms.gle/XAzR1pQBZiftvPQM7" target="_blank">survey</a> and we got <strong>308</strong> responses.
+Let’s see who Airflow users are, how they play with it, and what they miss.</p>
+
+<h1 id="overview-of-the-user">Overview of the user</h1>
+
+<p><strong>What best describes your current occupation?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Data Engineer</td>
+<td>194</td>
+<td>62.99%</td>
+</tr>
+
+<tr>
+<td>Developer</td>
+<td>34</td>
+<td>11.04%</td>
+</tr>
+
+<tr>
+<td>Architect</td>
+<td>23</td>
+<td>7.47%</td>
+</tr>
+
+<tr>
+<td>Data Scientist</td>
+<td>19</td>
+<td>6.17%</td>
+</tr>
+
+<tr>
+<td>Data Analyst</td>
+<td>13</td>
+<td>4.22%</td>
+</tr>
+
+<tr>
+<td>DevOps</td>
+<td>13</td>
+<td>4.22%</td>
+</tr>
+
+<tr>
+<td>IT Administrator</td>
+<td>2</td>
+<td>0.65%</td>
+</tr>
+
+<tr>
+<td>Machine Learning Engineer</td>
+<td>2</td>
+<td>0.65%</td>
+</tr>
+
+<tr>
+<td>Manager</td>
+<td>2</td>
+<td>0.65%</td>
+</tr>
+
+<tr>
+<td>Operations</td>
+<td>2</td>
+<td>0.65%</td>
+</tr>
+
+<tr>
+<td>Chief Data Officer</td>
+<td>1</td>
+<td>0.32%</td>
+</tr>
+
+<tr>
+<td>Engineering Manager</td>
+<td>1</td>
+<td>0.32%</td>
+</tr>
+
+<tr>
+<td>Intern</td>
+<td>1</td>
+<td>0.32%</td>
+</tr>
+
+<tr>
+<td>Product owner</td>
+<td>1</td>
+<td>0.32%</td>
+</tr>
+
+<tr>
+<td>Quant</td>
+<td>1</td>
+<td>0.32%</td>
+</tr>
+</tbody>
+</table>
+
+<p><strong>In your day to day job, what do you use Airflow for?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Data processing (ETL)</td>
+<td>298</td>
+<td>96.75%</td>
+</tr>
+
+<tr>
+<td>Artificial Intelligence and Machine Learning Pipelines</td>
+<td>90</td>
+<td>29.22%</td>
+</tr>
+
+<tr>
+<td>Automating DevOps operations</td>
+<td>64</td>
+<td>20.78%</td>
+</tr>
+</tbody>
+</table>
+
+<p>According to the survey, most of the Airflow users are the “data” people. Moreover,
+28.57% uses Airflow to both ETL and ML pipelines meaning that those two fields
+are somehow connected. Only five respondents use Airflow for DevOps operations only,
+That means that other 59 people who use Airflow for DevOps stuff use it also for
+ETL / ML purposes.</p>
+
+<p><strong>How many active DAGs do you have in your largest Airflow instance?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>0-20</td>
+<td>115</td>
+<td>37.34%</td>
+</tr>
+
+<tr>
+<td>21-40</td>
+<td>65</td>
+<td>21.10%</td>
+</tr>
+
+<tr>
+<td>41-60</td>
+<td>44</td>
+<td>14.29%</td>
+</tr>
+
+<tr>
+<td>61-100</td>
+<td>28</td>
+<td>9.09%</td>
+</tr>
+
+<tr>
+<td>101-200</td>
+<td>28</td>
+<td>9.09%</td>
+</tr>
+
+<tr>
+<td>201-300</td>
+<td>7</td>
+<td>2.27%</td>
+</tr>
+
+<tr>
+<td>301-999</td>
+<td>8</td>
+<td>2.60%</td>
+</tr>
+
+<tr>
+<td>1000+</td>
+<td>13</td>
+<td>4.22%</td>
+</tr>
+</tbody>
+</table>
+
+<p>The majority of users do not exceed 100 active DAGs per Airflow instance. However,
+as we can see there are users who exceed thousands of DAGs with a maximum number 5000.</p>
+
+<p><strong>What is the maximum number of tasks that you have used in one DAG?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>0-10</td>
+<td>61</td>
+<td>19.81%</td>
+</tr>
+
+<tr>
+<td>11-20</td>
+<td>60</td>
+<td>19.48%</td>
+</tr>
+
+<tr>
+<td>21-30</td>
+<td>31</td>
+<td>10.06%</td>
+</tr>
+
+<tr>
+<td>31-40</td>
+<td>21</td>
+<td>6.82%</td>
+</tr>
+
+<tr>
+<td>41-50</td>
+<td>26</td>
+<td>8.44%</td>
+</tr>
+
+<tr>
+<td>51-100</td>
+<td>36</td>
+<td>11.69%</td>
+</tr>
+
+<tr>
+<td>101-200</td>
+<td>28</td>
+<td>9.09%</td>
+</tr>
+
+<tr>
+<td>201-500</td>
+<td>21</td>
+<td>6.82%</td>
+</tr>
+
+<tr>
+<td>501+</td>
+<td>24</td>
+<td>11.54%</td>
+</tr>
+</tbody>
+</table>
+
+<p>The given maximum number of tasks in a single DAG was 10 000 (!). The number of tasks
+depends on the purposes of a DAG, so it’s rather hard to say if users have “simple”
+or “complicated” workflows.</p>
+
+<p><strong>When onboarding new members to Airflow, what is the biggest problem?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>No guide on best practises on developing DAGs</td>
+<td>160</td>
+<td>51.95%</td>
+</tr>
+
+<tr>
+<td>Small number of tutorials on different aspects of using Airflow</td>
+<td>57</td>
+<td>18.51%</td>
+</tr>
+
+<tr>
+<td>Documentation is not clear enough</td>
+<td>42</td>
+<td>13.64%</td>
+</tr>
+
+<tr>
+<td>Small number of blogs regarding Airflow</td>
+<td>6</td>
+<td>1.95%</td>
+</tr>
+
+<tr>
+<td>Other</td>
+<td>43</td>
+<td>13.96%</td>
+</tr>
+</tbody>
+</table>
+
+<p>This is an important result. Using Airflow is all about writing and scheduling DAGs.
+No guide or any other complete resource on best practices for developing Dags is a big
+problem. Diving deep in the “other” answers, we can find that:</p>
+
+<ul>
+<li>Airflow’s “magic” (scheduler, executors, schedule times) is hard to understand</li>
+<li>DAG testing is not easy to do and to explain</li>
+<li>Airflow UI needs some love.</li>
+</ul>
+
+<p><strong>How likely are you to recommend Apache Airflow?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Very Likely</td>
+<td>140</td>
+<td>45.45%</td>
+</tr>
+
+<tr>
+<td>Likely</td>
+<td>124</td>
+<td>40.26%</td>
+</tr>
+
+<tr>
+<td>Neutral</td>
+<td>33</td>
+<td>10.71%</td>
+</tr>
+
+<tr>
+<td>Unlikely</td>
+<td>8</td>
+<td>2.60%</td>
+</tr>
+
+<tr>
+<td>Very unlikely</td>
+<td>3</td>
+<td>0.97%</td>
+</tr>
+</tbody>
+</table>
+
+<p>This means that more than 85% of people who use Airflow like it. It seems Airflow does
+its job nicely. However, we have to remember that this survey is likely biased - it’s
+more likely that you respond to the survey if you like the tool you use. Should we
+focus then on those 11 people who did not like Airflow? It’s a good question.</p>
+
+<h2 id="airflow-usage">Airflow usage</h2>
+
+<p><strong>Which interface(s) of Airflow do you use as part of your current role?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Original Airflow Graphical User Interface</td>
+<td>297</td>
+<td>96.43%</td>
+</tr>
+
+<tr>
+<td>CLI</td>
+<td>126</td>
+<td>40.91%</td>
+</tr>
+
+<tr>
+<td>Original Airflow Graphical User Interface, CLI</td>
+<td>117</td>
+<td>37.99%</td>
+</tr>
+
+<tr>
+<td>API</td>
+<td>60</td>
+<td>19.48%</td>
+</tr>
+
+<tr>
+<td>Original Airflow Graphical User Interface, CLI, API</td>
+<td>32</td>
+<td>10.39%</td>
+</tr>
+
+<tr>
+<td>Custom (own created) Airflow Graphical User Interface</td>
+<td>25</td>
+<td>8.12%</td>
+</tr>
+</tbody>
+</table>
+
+<p>It’s visible that usage of CLI goes in pair with using Airflow web UI. Our
+survey included some UX related questions to allow us to understand how users
+use Airflow webserver.</p>
+
+<p><strong>What do you use the Graphical User Interface for?</strong></p>
+
+<p><img src="plot1.png" alt="" /></p>
+
+<p><strong>What do you use CLI for?</strong></p>
+
+<p><img src="plot2.png" alt="" /></p>
+
+<p><strong>In Airflow, which UI view(s) are important for you?</strong></p>
+
+<p><img src="plot3.png" alt="" /></p>
+
+<p>Here we see that the majority uses Web UI mostly for monitoring purposes:</p>
+
+<ul>
+<li>Monitoring DAGs</li>
+<li>Accessing logs</li>
+</ul>
+
+<p>An interesting result is that many people seem not to use backfilling as
+there’s no other way than to do it by CLI.</p>
+
+<p><strong>What executor type do you use?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Celery</td>
+<td>138</td>
+<td>44.81%</td>
+</tr>
+
+<tr>
+<td>Local</td>
+<td>85</td>
+<td>27.60%</td>
+</tr>
+
+<tr>
+<td>Kubernetes</td>
+<td>52</td>
+<td>16.88%</td>
+</tr>
+
+<tr>
+<td>Sequential</td>
+<td>22</td>
+<td>7.14%</td>
+</tr>
+
+<tr>
+<td>Other</td>
+<td>11</td>
+<td>3.57</td>
+</tr>
+</tbody>
+</table>
+
+<p>The other option mostly consisted of information that someone uses a few types or is
+migrating from one executor to another. What can be observed is an increase in usage
+of Local and Kubernetes executors when compared to results from an earlier <a href="https://ash.berlintaylor.com/writings/2019/02/airflow-user-survey-2019/" target="_blank">survey done
+by Ash</a>.</p>
+
+<p><strong>Do you use Kubernetes-based deployments for Airflow?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>No - we do not plan to use Kubernetes near term</td>
+<td>88</td>
+<td>28.57%</td>
+</tr>
+
+<tr>
+<td>Yes - setup on our own via Helm Chart or similar</td>
+<td>65</td>
+<td>21.10%</td>
+</tr>
+
+<tr>
+<td>Not yet - but we use Kubernetes in our organization and we could move</td>
+<td>61</td>
+<td>19.81%</td>
+</tr>
+
+<tr>
+<td>Yes - via managed service in the cloud (Composer / Astronomer etc.)</td>
+<td>45</td>
+<td>14.61%</td>
+</tr>
+
+<tr>
+<td>Not yet - but we plan to deploy Kubernetes in our organization soon</td>
+<td>42</td>
+<td>13.64%</td>
+</tr>
+
+<tr>
+<td>Other</td>
+<td>7</td>
+<td>2.27%</td>
+</tr>
+</tbody>
+</table>
+
+<p>The most interesting thing is that there’s nearly 30% of users who do not use Kubernetes,
+and they are not going to move. This means we should keep other deployment options in
+mind when working on Airflow 2.0. On the other hand, almost 70% of the users already
+use Kubernetes, or it’s a viable option for them.</p>
+
+<p><strong>Do you combine multiple DAGs?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>No, I don&rsquo;t combine multiple DAGs</td>
+<td>127</td>
+<td>41.23%</td>
+</tr>
+
+<tr>
+<td>Yes, through SubDAG</td>
+<td>73</td>
+<td>23.70%</td>
+</tr>
+
+<tr>
+<td>Yes, by triggering another DAG</td>
+<td>72</td>
+<td>23.38%</td>
+</tr>
+
+<tr>
+<td>Other</td>
+<td>36</td>
+<td>11.69%</td>
+</tr>
+</tbody>
+</table>
+
+<p>In the other category, 9 people explicitly mentioned using <code>ExternalTaskSensor</code>,
+and I think it could be treated as running subDAGs by triggering other DAGs.</p>
+
+<p><strong>Do you use Airflow Plugins? If yes, what do you use it for?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Adding new operators/sensors and hooks</td>
+<td>187</td>
+<td>60.71%</td>
+</tr>
+
+<tr>
+<td>I don&rsquo;t use Airflow plugins</td>
+<td>109</td>
+<td>35.39%</td>
+</tr>
+
+<tr>
+<td>Adding AppBuilder views &amp; menu items</td>
+<td>31</td>
+<td>10.06%</td>
+</tr>
+
+<tr>
+<td>Adding new executor</td>
+<td>18</td>
+<td>5.84%</td>
+</tr>
+
+<tr>
+<td>Adding OperatorExtraLinks</td>
+<td>7</td>
+<td>2.27%</td>
+</tr>
+</tbody>
+</table>
+
+<p>The high percentage - 60% for “Adding new operators/sensors and hooks” is quite a
+surprising result for some of us - especially that you do not actually need to use the
+plugin mechanism to add any of those. Those are standard python objects, and you can
+simply drop your hooks/operators/sensors code to <code>PYTHONPATH</code> environment variable and
+they will work. It seems that this may be a result of a lack of best practices guide.</p>
+
+<p>Plugins are more useful for adding views and menu items - yet only 10%.
+OperatorExtraLinks are even more useful (though relatively new) feature, so it’s not
+entirely surprising they are hardly used.</p>
+
+<p>It was also kind of surprising that someone at all uses plugins to use their own
+executors. We considered removing that option recently - but now we have to rethink
+our approach.</p>
+
+<p><strong>What metrics do you use to monitor Airflow?</strong></p>
+
+<p>There were a lot of different responses. Some use Prometheus and other services,
+others do not use any monitoring. One of the interesting responses linked to this
+solution for <a href="https://github.com/mastak/airflow_operators_metrics" target="_blank">airflow_operators_metrics</a>.</p>
+
+<h2 id="external-services">External services</h2>
+
+<p><strong>What external services do you use in your Airflow DAGs?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Amazon Web Services</td>
+<td>160</td>
+<td>51.95%</td>
+</tr>
+
+<tr>
+<td>Internal company systems</td>
+<td>150</td>
+<td>48.7%</td>
+</tr>
+
+<tr>
+<td>Hadoop / Spark / Flink / Other Apache software</td>
+<td>119</td>
+<td>38.64%</td>
+</tr>
+
+<tr>
+<td>Google Cloud Platform / Google APIs</td>
+<td>112</td>
+<td>36.36%</td>
+</tr>
+
+<tr>
+<td>Microsoft Azure</td>
+<td>28</td>
+<td>9.09%</td>
+</tr>
+
+<tr>
+<td>I do not use external services in my Airflow DAGs</td>
+<td>18</td>
+<td>5.84%</td>
+</tr>
+</tbody>
+</table>
+
+<p>It’s not surprising that Amazon Web Services is leading the way as they are considered the most mature
+cloud provider. Internal system and other Apache products on the next two positions are
+quite understandable if we take into account that the majority uses Airflow for ETL processes.</p>
+
+<p><strong>What external services do you use in your Airflow DAGs? (Mixed providers)</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Google Cloud Platform / Google APIs, Amazon Web Services</td>
+<td>44</td>
+<td>14.29%</td>
+</tr>
+
+<tr>
+<td>Amazon Web Services, Microsoft Azure</td>
+<td>5</td>
+<td>1.62%</td>
+</tr>
+
+<tr>
+<td>Google Cloud Platform / Google APIs, Microsoft Azure</td>
+<td>4</td>
+<td>1.3%</td>
+</tr>
+</tbody>
+</table>
+
+<p>This result is not surprising because companies usually prefer to stick with one cloud
+provider.</p>
+
+<p><strong>How do you integrate with external services?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Using Bash / Python operator</td>
+<td>220</td>
+<td>71.43%</td>
+</tr>
+
+<tr>
+<td>Using existing, dedicated operators / hooks</td>
+<td>217</td>
+<td>70.45%</td>
+</tr>
+
+<tr>
+<td>Using own, custom operators / hooks</td>
+<td>216</td>
+<td>70.13%</td>
+</tr>
+</tbody>
+</table>
+
+<p>We had some anecdotal evidence that people use more Python/Bash operators than the
+dedicated ones - but it looks like all ways of using Airflow to connect to external
+services are equally popular.</p>
+
+<h2 id="what-can-be-improved">What can be improved</h2>
+
+<p><strong>In your opinion, what could be improved in Airflow?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Scheduler performance</td>
+<td>189</td>
+<td>61.36%</td>
+</tr>
+
+<tr>
+<td>Web UI</td>
+<td>180</td>
+<td>58.44%</td>
+</tr>
+
+<tr>
+<td>Logging, monitoring and alerting</td>
+<td>145</td>
+<td>47.08%</td>
+</tr>
+
+<tr>
+<td>Examples, how-to, onboarding documentation</td>
+<td>143</td>
+<td>46.43%</td>
+</tr>
+
+<tr>
+<td>Technical documentation</td>
+<td>137</td>
+<td>44.48%</td>
+</tr>
+
+<tr>
+<td>Reliability</td>
+<td>112</td>
+<td>36.36%</td>
+</tr>
+
+<tr>
+<td>REST API</td>
+<td>96</td>
+<td>31.17%</td>
+</tr>
+
+<tr>
+<td>Authentication and authorization</td>
+<td>89</td>
+<td>28.9%</td>
+</tr>
+
+<tr>
+<td>External integration e.g. AWS, GCP, Apache product</td>
+<td>49</td>
+<td>15.91%</td>
+</tr>
+
+<tr>
+<td>CLI</td>
+<td>41</td>
+<td>13.31%</td>
+</tr>
+
+<tr>
+<td>I don’t know</td>
+<td>5</td>
+<td>1.62%</td>
+</tr>
+</tbody>
+</table>
+
+<p>The results are rather quite self-explaining. Improved performance of Airflow, better
+UI, and more telemetry are desirable. But this should go in pair with improved
+documentation and resources about using the Airflow, especially when we
+take into account the problem of onboarding new users.</p>
+
+<p>Another interesting point from that question is that only 16% think that operators
+should be extended and improved. This suggests that we should focus on improving
+Airflow core instead of adding more and more integrations.</p>
+
+<p><strong>What would be the most interesting feature for you?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Production-ready Airflow docker image</td>
+<td>175</td>
+<td>56.82%</td>
+</tr>
+
+<tr>
+<td>Declarative way of writing DAGs / automated DAGs generation</td>
+<td>155</td>
+<td>50.32%</td>
+</tr>
+
+<tr>
+<td>Horizontal Autoscaling</td>
+<td>122</td>
+<td>39.61%</td>
+</tr>
+
+<tr>
+<td>Asynchronous Operators</td>
+<td>97</td>
+<td>31.49%</td>
+</tr>
+
+<tr>
+<td>Stateless web server</td>
+<td>81</td>
+<td>26.3%</td>
+</tr>
+
+<tr>
+<td>Knative Executor</td>
+<td>48</td>
+<td>15.58%</td>
+</tr>
+
+<tr>
+<td>I already have all I need</td>
+<td>13</td>
+<td>4.22%</td>
+</tr>
+</tbody>
+</table>
+
+<p>Production Docker image wins, and it’s not a surprise. We all know that deploying
+Airflow is not a plug and play process, and that’s why the official image is being
+worked on by Jarek Potiuk. An unexpected result is that half of the users would like to
+have a declarative way of creating DAGs. That seems to be something that is “against Airflow”
+as we always emphasize the possibility of writing workflows in pure python. Stories
+about DAG generators are not new and confirm that there’s a need for a way to
+declare DAGs.</p>
+
+<h2 id="data">Data</h2>
+
+<p>If you think I missed something and you want to look for insights on your own the data is available
+for you here:</p>
+
+<ul>
+<li>Original data: <a href="https://storage.googleapis.com/airflow-survey/survey.csv" target="_blank">https://storage.googleapis.com/airflow-survey/survey.csv</a></li>
+<li>Processed: <a href="https://storage.googleapis.com/airflow-survey/airflow_survey_processed.csv" target="_blank">https://storage.googleapis.com/airflow-survey/airflow_survey_processed.csv</a></li>
+</ul>
+
+<p>The processed data includes multi-choice options one-hot encoded. If you find any interesting
+insight, please update the article (<a href="https://github.com/apache/airflow-site/blob/aip-11/CONTRIBUTE.md" target="_blank">make PR</a>
+to Airflow site).</p>
+
+ </description>
+ </item>
+
+ </channel>
+</rss>
diff --git a/categories/index.html b/categories/index.html
index fa5580d..93f2503 100644
--- a/categories/index.html
+++ b/categories/index.html
@@ -77,15 +77,15 @@ if (!doNotTrack) {
- <link rel="preload" href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" as="style">
- <link href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" rel="stylesheet" integrity="">
+ <link rel="preload" href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" as="style">
+ <link href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" rel="stylesheet" integrity="">
-<link rel="preload" href="/main.b2992.js" as="script">
-<link rel="preload" href="/chunk-1.b2992.js" as="script">
+<link rel="preload" href="/main.e3a02.js" as="script">
+<link rel="preload" href="/chunk-1.e3a02.js" as="script">
-<link rel="preload" href="/chunk-4.b2992.js" as="script">
+<link rel="preload" href="/chunk-4.e3a02.js" as="script">
@@ -548,6 +548,6 @@ if (!doNotTrack) {
</body>
- <script src="/main.b2992.js"></script>
+ <script src="/main.e3a02.js"></script>
</html>
diff --git a/chunk-1.e3a02.js b/chunk-1.e3a02.js
new file mode 100644
index 0000000..ecb6c9d
--- /dev/null
+++ b/chunk-1.e3a02.js
@@ -0,0 +1 @@
+(window.webpackJsonp=window.webpackJsonp||[]).push([[1],{60:function(t,e,n){"use strict";n.r(e);var r=n(58),i=n(0),l=n.n(i);function b(t){return function(t){if(Array.isArray(t)){for(var e=0,n=new Array(t.length);e<t.length;e++)n[e]=t[e];return n}}(t)||function(t){if(Symbol.iterator in Object(t)||"[object Arguments]"===Object.prototype.toString.call(t))return Array.from(t)}(t)||function(){throw new TypeError("Invalid attempt to spread non-iterable instance")}()}function c(t,e){if(!(t inst [...]
\ No newline at end of file
diff --git a/chunk-3.e3a02.js b/chunk-3.e3a02.js
new file mode 100644
index 0000000..80877c5
--- /dev/null
+++ b/chunk-3.e3a02.js
@@ -0,0 +1 @@
+(window.webpackJsonp=window.webpackJsonp||[]).push([[3],{59:function(e,t,n){"use strict";n.r(t),function(e){var t=n(57),a=n.n(t);function c(e,t){return function(e){if(Array.isArray(e))return e}(e)||function(e,t){if(!(Symbol.iterator in Object(e)||"[object Arguments]"===Object.prototype.toString.call(e)))return;var n=[],r=!0,o=!1,i=void 0;try{for(var a,c=e[Symbol.iterator]();!(r=(a=c.next()).done)&&(n.push(a.value),!t||n.length!==t);r=!0);}catch(e){o=!0,i=e}finally{try{r||null==c.return|| [...]
\ No newline at end of file
diff --git a/chunk-4.e3a02.js b/chunk-4.e3a02.js
new file mode 100644
index 0000000..b7d393c
--- /dev/null
+++ b/chunk-4.e3a02.js
@@ -0,0 +1,19 @@
+(window.webpackJsonp=window.webpackJsonp||[]).push([[4],{58:function(t,e,n){(function(e,_){t.exports=function o(i,r,s){function l(n,e){if(!r[n]){if(!i[n]){if(p)return p(n,!0);var t=new Error("Cannot find module '"+n+"'");throw t.code="MODULE_NOT_FOUND",t}var a=r[n]={exports:{}};i[n][0].call(a.exports,function(e){var t=i[n][1][e];return l(t||e)},a,a.exports,o,i,r,s)}return r[n].exports}for(var p=!1,e=0;e<s.length;e++)l(s[e]);return l}({1:[function(e,t,n){t.exports={project:{name:"p5",desc [...]
+/*!
+ * The buffer module from node.js, for the browser.
+ *
+ * @author Feross Aboukhadijeh <https://feross.org>
+ * @license MIT
+ */
+"use strict";var a=e("base64-js"),i=e("ieee754");n.Buffer=c,n.SlowBuffer=function(e){return+e!=e&&(e=0),c.alloc(+e)},n.INSPECT_MAX_BYTES=50;var o=2147483647;function r(e){if(o<e)throw new RangeError('The value "'+e+'" is invalid for option "size"');var t=new Uint8Array(e);return t.__proto__=c.prototype,t}function c(e,t,n){if("number"!=typeof e)return s(e,t,n);if("string"==typeof t)throw new TypeError('The "string" argument must be of type string. Received type number');return p(e)}functi [...]
+/*!
+ * @overview es6-promise - a tiny implementation of Promises/A+.
+ * @copyright Copyright (c) 2014 Yehuda Katz, Tom Dale, Stefan Penner and contributors (Conversion to ES6 API by Jake Archibald)
+ * @license Licensed under MIT license
+ * See https://raw.githubusercontent.com/stefanpenner/es6-promise/master/LICENSE
+ * @version v4.2.6+9869a4bc
+ */
+!function(e,t){"object"==typeof a&&void 0!==n?n.exports=t():e.ES6Promise=t()}(this,function(){"use strict";function p(e){return"function"==typeof e}var n=Array.isArray?Array.isArray:function(e){return"[object Array]"===Object.prototype.toString.call(e)},a=0,t=void 0,o=void 0,r=function(e,t){u[a]=e,u[a+1]=t,2===(a+=2)&&(o?o(h):m())},e="undefined"!=typeof window?window:void 0,i=e||{},s=i.MutationObserver||i.WebKitMutationObserver,l="undefined"==typeof self&&void 0!==O&&"[object process]"== [...]
+/*! @source http://purl.eligrey.com/github/FileSaver.js/blob/master/FileSaver.js */
+var a=a||function(l){"use strict";if(!(void 0===l||"undefined"!=typeof navigator&&/MSIE [1-9]\./.test(navigator.userAgent))){var e=l.document,p=function(){return l.URL||l.webkitURL||l},d=e.createElementNS("http://www.w3.org/1999/xhtml","a"),c="download"in d,u=/constructor/i.test(l.HTMLElement)||l.safari,h=/CriOS\/[\d]+/.test(navigator.userAgent),m=function(e){(l.setImmediate||l.setTimeout)(function(){throw e},0)},f=function(e){setTimeout(function(){"string"==typeof e?p().revokeObjectURL( [...]
+/*! https://mths.be/codepointat v0.2.0 by @mathias */String.prototype.codePointAt||function(){var e=function(){try{var e={},t=Object.defineProperty,n=t(e,e,e)&&t}catch(e){}return n}(),t=function(e){if(null==this)throw TypeError();var t=String(this),n=t.length,a=e?Number(e):0;if(a!=a&&(a=0),!(a<0||n<=a)){var o,i=t.charCodeAt(a);return 55296<=i&&i<=56319&&a+1<n&&56320<=(o=t.charCodeAt(a+1))&&o<=57343?1024*(i-55296)+o-56320+65536:i}};e?e(String.prototype,"codePointAt",{value:t,configurable: [...]
\ No newline at end of file
diff --git a/chunk-5.e3a02.js b/chunk-5.e3a02.js
new file mode 100644
index 0000000..d2d1bf3
--- /dev/null
+++ b/chunk-5.e3a02.js
@@ -0,0 +1,51 @@
+(window.webpackJsonp=window.webpackJsonp||[]).push([[5],{57:function(n,s,o){var a,u;
+/**
+ * lunr - http://lunrjs.com - A bit like Solr, but much smaller and not as bright - 2.3.8
+ * Copyright (C) 2019 Oliver Nightingale
+ * @license MIT
+ */!function(){var t,l,c,e,r,h,d,f,p,y,m,g,x,v,w,Q,k,S,E,L,b,P,T,O,I,z=function(e){var t=new z.Builder;return t.pipeline.add(z.trimmer,z.stopWordFilter,z.stemmer),t.searchPipeline.add(z.stemmer),e.call(t,t),t.build()};function i(e){var t,r,i,n,s,o,a;if(e.length<3)return e;if("y"==(i=e.substr(0,1))&&(e=i.toUpperCase()+e.substr(1)),s=m,(n=y).test(e)?e=e.replace(n,"$1$2"):s.test(e)&&(e=e.replace(s,"$1$2")),s=x,(n=g).test(e)){var u=n.exec(e);(n=h).test(u[1])&&(n=v,e=e.replace(n,""))}else if( [...]
+/*!
+ * lunr.utils
+ * Copyright (C) 2019 Oliver Nightingale
+ */,z.utils={},z.utils.warn=(t=this,function(e){t.console&&console.warn&&console.warn(e)}),z.utils.asString=function(e){return null==e?"":e.toString()},z.utils.clone=function(e){if(null==e)return e;for(var t=Object.create(null),r=Object.keys(e),i=0;i<r.length;i++){var n=r[i],s=e[n];if(Array.isArray(s))t[n]=s.slice();else{if("string"!=typeof s&&"number"!=typeof s&&"boolean"!=typeof s)throw new TypeError("clone is not deep and does not support nested objects");t[n]=s}}return t},z.FieldRef= [...]
+/*!
+ * lunr.Set
+ * Copyright (C) 2019 Oliver Nightingale
+ */,z.Set=function(e){if(this.elements=Object.create(null),e){this.length=e.length;for(var t=0;t<this.length;t++)this.elements[e[t]]=!0}else this.length=0},z.Set.complete={intersect:function(e){return e},union:function(e){return e},contains:function(){return!0}},z.Set.empty={intersect:function(){return this},union:function(e){return e},contains:function(){return!1}},z.Set.prototype.contains=function(e){return!!this.elements[e]},z.Set.prototype.intersect=function(e){var t,r,i,n=[];if(e=== [...]
+/*!
+ * lunr.tokenizer
+ * Copyright (C) 2019 Oliver Nightingale
+ */,z.tokenizer=function(e,t){if(null==e||null==e)return[];if(Array.isArray(e))return e.map(function(e){return new z.Token(z.utils.asString(e).toLowerCase(),z.utils.clone(t))});for(var r=e.toString().toLowerCase(),i=r.length,n=[],s=0,o=0;s<=i;s++){var a=s-o;if(r.charAt(s).match(z.tokenizer.separator)||s==i){if(0<a){var u=z.utils.clone(t)||{};u.position=[o,a],u.index=n.length,n.push(new z.Token(r.slice(o,s),u))}o=s+1}}return n},z.tokenizer.separator=/[\s\-]+/
+/*!
+ * lunr.Pipeline
+ * Copyright (C) 2019 Oliver Nightingale
+ */,z.Pipeline=function(){this._stack=[]},z.Pipeline.registeredFunctions=Object.create(null),z.Pipeline.registerFunction=function(e,t){t in this.registeredFunctions&&z.utils.warn("Overwriting existing registered function: "+t),e.label=t,z.Pipeline.registeredFunctions[e.label]=e},z.Pipeline.warnIfFunctionNotRegistered=function(e){e.label&&e.label in this.registeredFunctions||z.utils.warn("Function is not registered with pipeline. This may cause problems when serialising the index.\n",e)}, [...]
+/*!
+ * lunr.Vector
+ * Copyright (C) 2019 Oliver Nightingale
+ */,z.Vector=function(e){this._magnitude=0,this.elements=e||[]},z.Vector.prototype.positionForIndex=function(e){if(0==this.elements.length)return 0;for(var t=0,r=this.elements.length/2,i=r-t,n=Math.floor(i/2),s=this.elements[2*n];1<i&&(s<e&&(t=n),e<s&&(r=n),s!=e);)i=r-t,n=t+Math.floor(i/2),s=this.elements[2*n];return s==e?2*n:e<s?2*n:s<e?2*(n+1):void 0},z.Vector.prototype.insert=function(e,t){this.upsert(e,t,function(){throw"duplicate index"})},z.Vector.prototype.upsert=function(e,t,r){t [...]
+/*!
+ * lunr.stemmer
+ * Copyright (C) 2019 Oliver Nightingale
+ * Includes code from - http://tartarus.org/~martin/PorterStemmer/js.txt
+ */,z.stemmer=(l={ational:"ate",tional:"tion",enci:"ence",anci:"ance",izer:"ize",bli:"ble",alli:"al",entli:"ent",eli:"e",ousli:"ous",ization:"ize",ation:"ate",ator:"ate",alism:"al",iveness:"ive",fulness:"ful",ousness:"ous",aliti:"al",iviti:"ive",biliti:"ble",logi:"log"},c={icate:"ic",ative:"",alize:"al",iciti:"ic",ical:"ic",ful:"",ness:""},e="[aeiouy]",r="[^aeiou][^aeiouy]*",h=new RegExp("^([^aeiou][^aeiouy]*)?[aeiouy][aeiou]*[^aeiou][^aeiouy]*"),d=new RegExp("^([^aeiou][^aeiouy]*)?[aeio [...]
+/*!
+ * lunr.stopWordFilter
+ * Copyright (C) 2019 Oliver Nightingale
+ */,z.generateStopWordFilter=function(e){var t=e.reduce(function(e,t){return e[t]=t,e},{});return function(e){if(e&&t[e.toString()]!==e.toString())return e}},z.stopWordFilter=z.generateStopWordFilter(["a","able","about","across","after","all","almost","also","am","among","an","and","any","are","as","at","be","because","been","but","by","can","cannot","could","dear","did","do","does","either","else","ever","every","for","from","get","got","had","has","have","he","her","hers","him","his"," [...]
+/*!
+ * lunr.trimmer
+ * Copyright (C) 2019 Oliver Nightingale
+ */,z.trimmer=function(e){return e.update(function(e){return e.replace(/^\W+/,"").replace(/\W+$/,"")})},z.Pipeline.registerFunction(z.trimmer,"trimmer")
+/*!
+ * lunr.TokenSet
+ * Copyright (C) 2019 Oliver Nightingale
+ */,z.TokenSet=function(){this.final=!1,this.edges={},this.id=z.TokenSet._nextId,z.TokenSet._nextId+=1},z.TokenSet._nextId=1,z.TokenSet.fromArray=function(e){for(var t=new z.TokenSet.Builder,r=0,i=e.length;r<i;r++)t.insert(e[r]);return t.finish(),t.root},z.TokenSet.fromClause=function(e){return"editDistance"in e?z.TokenSet.fromFuzzyString(e.term,e.editDistance):z.TokenSet.fromString(e.term)},z.TokenSet.fromFuzzyString=function(e,t){for(var r=new z.TokenSet,i=[{node:r,editsRemaining:t,str [...]
+/*!
+ * lunr.Index
+ * Copyright (C) 2019 Oliver Nightingale
+ */,z.Index=function(e){this.invertedIndex=e.invertedIndex,this.fieldVectors=e.fieldVectors,this.tokenSet=e.tokenSet,this.fields=e.fields,this.pipeline=e.pipeline},z.Index.prototype.search=function(t){return this.query(function(e){new z.QueryParser(t,e).parse()})},z.Index.prototype.query=function(e){for(var t=new z.Query(this.fields),r=Object.create(null),i=Object.create(null),n=Object.create(null),s=Object.create(null),o=Object.create(null),a=0;a<this.fields.length;a++)i[this.fields[a]] [...]
+/*!
+ * lunr.Builder
+ * Copyright (C) 2019 Oliver Nightingale
+ */,z.Builder=function(){this._ref="id",this._fields=Object.create(null),this._documents=Object.create(null),this.invertedIndex=Object.create(null),this.fieldTermFrequencies={},this.fieldLengths={},this.tokenizer=z.tokenizer,this.pipeline=new z.Pipeline,this.searchPipeline=new z.Pipeline,this.documentCount=0,this._b=.75,this._k1=1.2,this.termIndex=0,this.metadataWhitelist=[]},z.Builder.prototype.ref=function(e){this._ref=e},z.Builder.prototype.field=function(e,t){if(/\//.test(e))throw ne [...]
\ No newline at end of file
diff --git a/community/index.html b/community/index.html
index 1dee48f..d3c9189 100644
--- a/community/index.html
+++ b/community/index.html
@@ -77,15 +77,15 @@ if (!doNotTrack) {
- <link rel="preload" href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" as="style">
- <link href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" rel="stylesheet" integrity="">
+ <link rel="preload" href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" as="style">
+ <link href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" rel="stylesheet" integrity="">
-<link rel="preload" href="/main.b2992.js" as="script">
-<link rel="preload" href="/chunk-1.b2992.js" as="script">
+<link rel="preload" href="/main.e3a02.js" as="script">
+<link rel="preload" href="/chunk-1.e3a02.js" as="script">
-<link rel="preload" href="/chunk-4.b2992.js" as="script">
+<link rel="preload" href="/chunk-4.e3a02.js" as="script">
@@ -1666,6 +1666,6 @@ if (!doNotTrack) {
</body>
- <script src="/main.b2992.js"></script>
+ <script src="/main.e3a02.js"></script>
</html>
diff --git a/docs.e3a02.js b/docs.e3a02.js
new file mode 100644
index 0000000..4872e89
--- /dev/null
+++ b/docs.e3a02.js
@@ -0,0 +1 @@
+!function(r){var n={};function o(t){if(n[t])return n[t].exports;var e=n[t]={i:t,l:!1,exports:{}};return r[t].call(e.exports,e,e.exports,o),e.l=!0,e.exports}o.m=r,o.c=n,o.d=function(t,e,r){o.o(t,e)||Object.defineProperty(t,e,{enumerable:!0,get:r})},o.r=function(t){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(t,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(t,"__esModule",{value:!0})},o.t=function(e,t){if(1&t&&(e=o(e)),8&t)return e;if(4&t&&"object"==typ [...]
\ No newline at end of file
diff --git a/index.html b/index.html
index 179d547..3189a68 100644
--- a/index.html
+++ b/index.html
@@ -77,15 +77,15 @@ if (!doNotTrack) {
- <link rel="preload" href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" as="style">
- <link href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" rel="stylesheet" integrity="">
+ <link rel="preload" href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" as="style">
+ <link href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" rel="stylesheet" integrity="">
-<link rel="preload" href="/main.b2992.js" as="script">
-<link rel="preload" href="/chunk-1.b2992.js" as="script">
+<link rel="preload" href="/main.e3a02.js" as="script">
+<link rel="preload" href="/chunk-1.e3a02.js" as="script">
-<link rel="preload" href="/chunk-4.b2992.js" as="script">
+<link rel="preload" href="/chunk-4.e3a02.js" as="script">
@@ -1170,12 +1170,12 @@ if (!doNotTrack) {
<div id="integrations-container" class="list-items">
- <a class="list-item" href="/docs/stable/integration.html#service-integrations">
+ <a class="list-item" href="/docs/stable/integration.html#aws-amazon-web-services">
<div class="card">
<div class="box-event box-event__integration">
- <span class="box-event__integration--name">Google Campaign Manager</span>
+ <span class="box-event__integration--name">Amazon EMR</span>
</div>
</div>
@@ -1183,12 +1183,12 @@ if (!doNotTrack) {
- <a class="list-item" href="/docs/stable/integration.html#protocol-integrations">
+ <a class="list-item" href="/docs/stable/integration.html#gcp-google-cloud-platform">
<div class="card">
<div class="box-event box-event__integration">
- <span class="box-event__integration--name">Simple Mail Transfer Protocol (SMTP)</span>
+ <span class="box-event__integration--name">Cloud Pub/Sub</span>
</div>
</div>
@@ -1196,12 +1196,12 @@ if (!doNotTrack) {
- <a class="list-item" href="/docs/stable/integration.html#service-integrations">
+ <a class="list-item" href="/docs/stable/integration.html#aws-amazon-web-services">
<div class="card">
<div class="box-event box-event__integration">
- <span class="box-event__integration--name">Atlassian Jira</span>
+ <span class="box-event__integration--name">Amazon Redshift</span>
</div>
</div>
@@ -1209,12 +1209,12 @@ if (!doNotTrack) {
- <a class="list-item" href="/docs/stable/integration.html#software-integrations">
+ <a class="list-item" href="/docs/stable/integration.html#asf-apache-software-foundation">
<div class="card">
<div class="box-event box-event__integration">
- <span class="box-event__integration--name">Presto</span>
+ <span class="box-event__integration--name">WebHDFS</span>
</div>
</div>
@@ -1222,12 +1222,12 @@ if (!doNotTrack) {
- <a class="list-item" href="/docs/stable/integration.html#aws-amazon-web-services">
+ <a class="list-item" href="/docs/stable/integration.html#service-integrations">
<div class="card">
<div class="box-event box-event__integration">
- <span class="box-event__integration--name">AWS Lambda</span>
+ <span class="box-event__integration--name">Salesforce</span>
</div>
</div>
@@ -1240,7 +1240,7 @@ if (!doNotTrack) {
<div class="card">
<div class="box-event box-event__integration">
- <span class="box-event__integration--name">Microsoft SQL Server (MSSQL)</span>
+ <span class="box-event__integration--name">GNU Bash</span>
</div>
</div>
@@ -1248,12 +1248,12 @@ if (!doNotTrack) {
- <a class="list-item" href="/docs/stable/integration.html#gcp-google-cloud-platform">
+ <a class="list-item" href="/docs/stable/integration.html#protocol-integrations">
<div class="card">
<div class="box-event box-event__integration">
- <span class="box-event__integration--name">Datastore</span>
+ <span class="box-event__integration--name">Windows Remote Management (WinRM)</span>
</div>
</div>
@@ -1261,12 +1261,12 @@ if (!doNotTrack) {
- <a class="list-item" href="/docs/stable/integration.html#software-integrations">
+ <a class="list-item" href="/docs/stable/integration.html#service-integrations">
<div class="card">
<div class="box-event box-event__integration">
- <span class="box-event__integration--name">Papermill</span>
+ <span class="box-event__integration--name">Snowflake</span>
</div>
</div>
@@ -1652,6 +1652,6 @@ if (!doNotTrack) {
</body>
- <script src="/main.b2992.js"></script>
+ <script src="/main.e3a02.js"></script>
</html>
diff --git a/index.xml b/index.xml
index 598bea6..5b96ac5 100644
--- a/index.xml
+++ b/index.xml
@@ -13,6 +13,1032 @@
<item>
+ <title>Blog: Airflow Survey 2019</title>
+ <link>/blog/airflow-survey/</link>
+ <pubDate>Wed, 11 Dec 2019 00:00:00 +0000</pubDate>
+
+ <guid>/blog/airflow-survey/</guid>
+ <description>
+
+
+
+
+<h1 id="apache-airflow-survey-2019">Apache Airflow Survey 2019</h1>
+
+<p>Apache Airflow is <a href="https://www.astronomer.io/blog/why-airflow/" target="_blank">growing faster than ever</a>.
+Thus, receiving and adjusting to our users’ feedback is a must. We created
+<a href="https://forms.gle/XAzR1pQBZiftvPQM7" target="_blank">survey</a> and we got <strong>308</strong> responses.
+Let’s see who Airflow users are, how they play with it, and what they miss.</p>
+
+<h1 id="overview-of-the-user">Overview of the user</h1>
+
+<p><strong>What best describes your current occupation?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Data Engineer</td>
+<td>194</td>
+<td>62.99%</td>
+</tr>
+
+<tr>
+<td>Developer</td>
+<td>34</td>
+<td>11.04%</td>
+</tr>
+
+<tr>
+<td>Architect</td>
+<td>23</td>
+<td>7.47%</td>
+</tr>
+
+<tr>
+<td>Data Scientist</td>
+<td>19</td>
+<td>6.17%</td>
+</tr>
+
+<tr>
+<td>Data Analyst</td>
+<td>13</td>
+<td>4.22%</td>
+</tr>
+
+<tr>
+<td>DevOps</td>
+<td>13</td>
+<td>4.22%</td>
+</tr>
+
+<tr>
+<td>IT Administrator</td>
+<td>2</td>
+<td>0.65%</td>
+</tr>
+
+<tr>
+<td>Machine Learning Engineer</td>
+<td>2</td>
+<td>0.65%</td>
+</tr>
+
+<tr>
+<td>Manager</td>
+<td>2</td>
+<td>0.65%</td>
+</tr>
+
+<tr>
+<td>Operations</td>
+<td>2</td>
+<td>0.65%</td>
+</tr>
+
+<tr>
+<td>Chief Data Officer</td>
+<td>1</td>
+<td>0.32%</td>
+</tr>
+
+<tr>
+<td>Engineering Manager</td>
+<td>1</td>
+<td>0.32%</td>
+</tr>
+
+<tr>
+<td>Intern</td>
+<td>1</td>
+<td>0.32%</td>
+</tr>
+
+<tr>
+<td>Product owner</td>
+<td>1</td>
+<td>0.32%</td>
+</tr>
+
+<tr>
+<td>Quant</td>
+<td>1</td>
+<td>0.32%</td>
+</tr>
+</tbody>
+</table>
+
+<p><strong>In your day to day job, what do you use Airflow for?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Data processing (ETL)</td>
+<td>298</td>
+<td>96.75%</td>
+</tr>
+
+<tr>
+<td>Artificial Intelligence and Machine Learning Pipelines</td>
+<td>90</td>
+<td>29.22%</td>
+</tr>
+
+<tr>
+<td>Automating DevOps operations</td>
+<td>64</td>
+<td>20.78%</td>
+</tr>
+</tbody>
+</table>
+
+<p>According to the survey, most of the Airflow users are the “data” people. Moreover,
+28.57% uses Airflow to both ETL and ML pipelines meaning that those two fields
+are somehow connected. Only five respondents use Airflow for DevOps operations only,
+That means that other 59 people who use Airflow for DevOps stuff use it also for
+ETL / ML purposes.</p>
+
+<p><strong>How many active DAGs do you have in your largest Airflow instance?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>0-20</td>
+<td>115</td>
+<td>37.34%</td>
+</tr>
+
+<tr>
+<td>21-40</td>
+<td>65</td>
+<td>21.10%</td>
+</tr>
+
+<tr>
+<td>41-60</td>
+<td>44</td>
+<td>14.29%</td>
+</tr>
+
+<tr>
+<td>61-100</td>
+<td>28</td>
+<td>9.09%</td>
+</tr>
+
+<tr>
+<td>101-200</td>
+<td>28</td>
+<td>9.09%</td>
+</tr>
+
+<tr>
+<td>201-300</td>
+<td>7</td>
+<td>2.27%</td>
+</tr>
+
+<tr>
+<td>301-999</td>
+<td>8</td>
+<td>2.60%</td>
+</tr>
+
+<tr>
+<td>1000+</td>
+<td>13</td>
+<td>4.22%</td>
+</tr>
+</tbody>
+</table>
+
+<p>The majority of users do not exceed 100 active DAGs per Airflow instance. However,
+as we can see there are users who exceed thousands of DAGs with a maximum number 5000.</p>
+
+<p><strong>What is the maximum number of tasks that you have used in one DAG?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>0-10</td>
+<td>61</td>
+<td>19.81%</td>
+</tr>
+
+<tr>
+<td>11-20</td>
+<td>60</td>
+<td>19.48%</td>
+</tr>
+
+<tr>
+<td>21-30</td>
+<td>31</td>
+<td>10.06%</td>
+</tr>
+
+<tr>
+<td>31-40</td>
+<td>21</td>
+<td>6.82%</td>
+</tr>
+
+<tr>
+<td>41-50</td>
+<td>26</td>
+<td>8.44%</td>
+</tr>
+
+<tr>
+<td>51-100</td>
+<td>36</td>
+<td>11.69%</td>
+</tr>
+
+<tr>
+<td>101-200</td>
+<td>28</td>
+<td>9.09%</td>
+</tr>
+
+<tr>
+<td>201-500</td>
+<td>21</td>
+<td>6.82%</td>
+</tr>
+
+<tr>
+<td>501+</td>
+<td>24</td>
+<td>11.54%</td>
+</tr>
+</tbody>
+</table>
+
+<p>The given maximum number of tasks in a single DAG was 10 000 (!). The number of tasks
+depends on the purposes of a DAG, so it’s rather hard to say if users have “simple”
+or “complicated” workflows.</p>
+
+<p><strong>When onboarding new members to Airflow, what is the biggest problem?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>No guide on best practises on developing DAGs</td>
+<td>160</td>
+<td>51.95%</td>
+</tr>
+
+<tr>
+<td>Small number of tutorials on different aspects of using Airflow</td>
+<td>57</td>
+<td>18.51%</td>
+</tr>
+
+<tr>
+<td>Documentation is not clear enough</td>
+<td>42</td>
+<td>13.64%</td>
+</tr>
+
+<tr>
+<td>Small number of blogs regarding Airflow</td>
+<td>6</td>
+<td>1.95%</td>
+</tr>
+
+<tr>
+<td>Other</td>
+<td>43</td>
+<td>13.96%</td>
+</tr>
+</tbody>
+</table>
+
+<p>This is an important result. Using Airflow is all about writing and scheduling DAGs.
+No guide or any other complete resource on best practices for developing Dags is a big
+problem. Diving deep in the “other” answers, we can find that:</p>
+
+<ul>
+<li>Airflow’s “magic” (scheduler, executors, schedule times) is hard to understand</li>
+<li>DAG testing is not easy to do and to explain</li>
+<li>Airflow UI needs some love.</li>
+</ul>
+
+<p><strong>How likely are you to recommend Apache Airflow?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Very Likely</td>
+<td>140</td>
+<td>45.45%</td>
+</tr>
+
+<tr>
+<td>Likely</td>
+<td>124</td>
+<td>40.26%</td>
+</tr>
+
+<tr>
+<td>Neutral</td>
+<td>33</td>
+<td>10.71%</td>
+</tr>
+
+<tr>
+<td>Unlikely</td>
+<td>8</td>
+<td>2.60%</td>
+</tr>
+
+<tr>
+<td>Very unlikely</td>
+<td>3</td>
+<td>0.97%</td>
+</tr>
+</tbody>
+</table>
+
+<p>This means that more than 85% of people who use Airflow like it. It seems Airflow does
+its job nicely. However, we have to remember that this survey is likely biased - it’s
+more likely that you respond to the survey if you like the tool you use. Should we
+focus then on those 11 people who did not like Airflow? It’s a good question.</p>
+
+<h2 id="airflow-usage">Airflow usage</h2>
+
+<p><strong>Which interface(s) of Airflow do you use as part of your current role?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Original Airflow Graphical User Interface</td>
+<td>297</td>
+<td>96.43%</td>
+</tr>
+
+<tr>
+<td>CLI</td>
+<td>126</td>
+<td>40.91%</td>
+</tr>
+
+<tr>
+<td>Original Airflow Graphical User Interface, CLI</td>
+<td>117</td>
+<td>37.99%</td>
+</tr>
+
+<tr>
+<td>API</td>
+<td>60</td>
+<td>19.48%</td>
+</tr>
+
+<tr>
+<td>Original Airflow Graphical User Interface, CLI, API</td>
+<td>32</td>
+<td>10.39%</td>
+</tr>
+
+<tr>
+<td>Custom (own created) Airflow Graphical User Interface</td>
+<td>25</td>
+<td>8.12%</td>
+</tr>
+</tbody>
+</table>
+
+<p>It’s visible that usage of CLI goes in pair with using Airflow web UI. Our
+survey included some UX related questions to allow us to understand how users
+use Airflow webserver.</p>
+
+<p><strong>What do you use the Graphical User Interface for?</strong></p>
+
+<p><img src="plot1.png" alt="" /></p>
+
+<p><strong>What do you use CLI for?</strong></p>
+
+<p><img src="plot2.png" alt="" /></p>
+
+<p><strong>In Airflow, which UI view(s) are important for you?</strong></p>
+
+<p><img src="plot3.png" alt="" /></p>
+
+<p>Here we see that the majority uses Web UI mostly for monitoring purposes:</p>
+
+<ul>
+<li>Monitoring DAGs</li>
+<li>Accessing logs</li>
+</ul>
+
+<p>An interesting result is that many people seem not to use backfilling as
+there’s no other way than to do it by CLI.</p>
+
+<p><strong>What executor type do you use?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Celery</td>
+<td>138</td>
+<td>44.81%</td>
+</tr>
+
+<tr>
+<td>Local</td>
+<td>85</td>
+<td>27.60%</td>
+</tr>
+
+<tr>
+<td>Kubernetes</td>
+<td>52</td>
+<td>16.88%</td>
+</tr>
+
+<tr>
+<td>Sequential</td>
+<td>22</td>
+<td>7.14%</td>
+</tr>
+
+<tr>
+<td>Other</td>
+<td>11</td>
+<td>3.57</td>
+</tr>
+</tbody>
+</table>
+
+<p>The other option mostly consisted of information that someone uses a few types or is
+migrating from one executor to another. What can be observed is an increase in usage
+of Local and Kubernetes executors when compared to results from an earlier <a href="https://ash.berlintaylor.com/writings/2019/02/airflow-user-survey-2019/" target="_blank">survey done
+by Ash</a>.</p>
+
+<p><strong>Do you use Kubernetes-based deployments for Airflow?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>No - we do not plan to use Kubernetes near term</td>
+<td>88</td>
+<td>28.57%</td>
+</tr>
+
+<tr>
+<td>Yes - setup on our own via Helm Chart or similar</td>
+<td>65</td>
+<td>21.10%</td>
+</tr>
+
+<tr>
+<td>Not yet - but we use Kubernetes in our organization and we could move</td>
+<td>61</td>
+<td>19.81%</td>
+</tr>
+
+<tr>
+<td>Yes - via managed service in the cloud (Composer / Astronomer etc.)</td>
+<td>45</td>
+<td>14.61%</td>
+</tr>
+
+<tr>
+<td>Not yet - but we plan to deploy Kubernetes in our organization soon</td>
+<td>42</td>
+<td>13.64%</td>
+</tr>
+
+<tr>
+<td>Other</td>
+<td>7</td>
+<td>2.27%</td>
+</tr>
+</tbody>
+</table>
+
+<p>The most interesting thing is that there’s nearly 30% of users who do not use Kubernetes,
+and they are not going to move. This means we should keep other deployment options in
+mind when working on Airflow 2.0. On the other hand, almost 70% of the users already
+use Kubernetes, or it’s a viable option for them.</p>
+
+<p><strong>Do you combine multiple DAGs?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>No, I don&rsquo;t combine multiple DAGs</td>
+<td>127</td>
+<td>41.23%</td>
+</tr>
+
+<tr>
+<td>Yes, through SubDAG</td>
+<td>73</td>
+<td>23.70%</td>
+</tr>
+
+<tr>
+<td>Yes, by triggering another DAG</td>
+<td>72</td>
+<td>23.38%</td>
+</tr>
+
+<tr>
+<td>Other</td>
+<td>36</td>
+<td>11.69%</td>
+</tr>
+</tbody>
+</table>
+
+<p>In the other category, 9 people explicitly mentioned using <code>ExternalTaskSensor</code>,
+and I think it could be treated as running subDAGs by triggering other DAGs.</p>
+
+<p><strong>Do you use Airflow Plugins? If yes, what do you use it for?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Adding new operators/sensors and hooks</td>
+<td>187</td>
+<td>60.71%</td>
+</tr>
+
+<tr>
+<td>I don&rsquo;t use Airflow plugins</td>
+<td>109</td>
+<td>35.39%</td>
+</tr>
+
+<tr>
+<td>Adding AppBuilder views &amp; menu items</td>
+<td>31</td>
+<td>10.06%</td>
+</tr>
+
+<tr>
+<td>Adding new executor</td>
+<td>18</td>
+<td>5.84%</td>
+</tr>
+
+<tr>
+<td>Adding OperatorExtraLinks</td>
+<td>7</td>
+<td>2.27%</td>
+</tr>
+</tbody>
+</table>
+
+<p>The high percentage - 60% for “Adding new operators/sensors and hooks” is quite a
+surprising result for some of us - especially that you do not actually need to use the
+plugin mechanism to add any of those. Those are standard python objects, and you can
+simply drop your hooks/operators/sensors code to <code>PYTHONPATH</code> environment variable and
+they will work. It seems that this may be a result of a lack of best practices guide.</p>
+
+<p>Plugins are more useful for adding views and menu items - yet only 10%.
+OperatorExtraLinks are even more useful (though relatively new) feature, so it’s not
+entirely surprising they are hardly used.</p>
+
+<p>It was also kind of surprising that someone at all uses plugins to use their own
+executors. We considered removing that option recently - but now we have to rethink
+our approach.</p>
+
+<p><strong>What metrics do you use to monitor Airflow?</strong></p>
+
+<p>There were a lot of different responses. Some use Prometheus and other services,
+others do not use any monitoring. One of the interesting responses linked to this
+solution for <a href="https://github.com/mastak/airflow_operators_metrics" target="_blank">airflow_operators_metrics</a>.</p>
+
+<h2 id="external-services">External services</h2>
+
+<p><strong>What external services do you use in your Airflow DAGs?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Amazon Web Services</td>
+<td>160</td>
+<td>51.95%</td>
+</tr>
+
+<tr>
+<td>Internal company systems</td>
+<td>150</td>
+<td>48.7%</td>
+</tr>
+
+<tr>
+<td>Hadoop / Spark / Flink / Other Apache software</td>
+<td>119</td>
+<td>38.64%</td>
+</tr>
+
+<tr>
+<td>Google Cloud Platform / Google APIs</td>
+<td>112</td>
+<td>36.36%</td>
+</tr>
+
+<tr>
+<td>Microsoft Azure</td>
+<td>28</td>
+<td>9.09%</td>
+</tr>
+
+<tr>
+<td>I do not use external services in my Airflow DAGs</td>
+<td>18</td>
+<td>5.84%</td>
+</tr>
+</tbody>
+</table>
+
+<p>It’s not surprising that Amazon Web Services is leading the way as they are considered the most mature
+cloud provider. Internal system and other Apache products on the next two positions are
+quite understandable if we take into account that the majority uses Airflow for ETL processes.</p>
+
+<p><strong>What external services do you use in your Airflow DAGs? (Mixed providers)</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Google Cloud Platform / Google APIs, Amazon Web Services</td>
+<td>44</td>
+<td>14.29%</td>
+</tr>
+
+<tr>
+<td>Amazon Web Services, Microsoft Azure</td>
+<td>5</td>
+<td>1.62%</td>
+</tr>
+
+<tr>
+<td>Google Cloud Platform / Google APIs, Microsoft Azure</td>
+<td>4</td>
+<td>1.3%</td>
+</tr>
+</tbody>
+</table>
+
+<p>This result is not surprising because companies usually prefer to stick with one cloud
+provider.</p>
+
+<p><strong>How do you integrate with external services?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Using Bash / Python operator</td>
+<td>220</td>
+<td>71.43%</td>
+</tr>
+
+<tr>
+<td>Using existing, dedicated operators / hooks</td>
+<td>217</td>
+<td>70.45%</td>
+</tr>
+
+<tr>
+<td>Using own, custom operators / hooks</td>
+<td>216</td>
+<td>70.13%</td>
+</tr>
+</tbody>
+</table>
+
+<p>We had some anecdotal evidence that people use more Python/Bash operators than the
+dedicated ones - but it looks like all ways of using Airflow to connect to external
+services are equally popular.</p>
+
+<h2 id="what-can-be-improved">What can be improved</h2>
+
+<p><strong>In your opinion, what could be improved in Airflow?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Scheduler performance</td>
+<td>189</td>
+<td>61.36%</td>
+</tr>
+
+<tr>
+<td>Web UI</td>
+<td>180</td>
+<td>58.44%</td>
+</tr>
+
+<tr>
+<td>Logging, monitoring and alerting</td>
+<td>145</td>
+<td>47.08%</td>
+</tr>
+
+<tr>
+<td>Examples, how-to, onboarding documentation</td>
+<td>143</td>
+<td>46.43%</td>
+</tr>
+
+<tr>
+<td>Technical documentation</td>
+<td>137</td>
+<td>44.48%</td>
+</tr>
+
+<tr>
+<td>Reliability</td>
+<td>112</td>
+<td>36.36%</td>
+</tr>
+
+<tr>
+<td>REST API</td>
+<td>96</td>
+<td>31.17%</td>
+</tr>
+
+<tr>
+<td>Authentication and authorization</td>
+<td>89</td>
+<td>28.9%</td>
+</tr>
+
+<tr>
+<td>External integration e.g. AWS, GCP, Apache product</td>
+<td>49</td>
+<td>15.91%</td>
+</tr>
+
+<tr>
+<td>CLI</td>
+<td>41</td>
+<td>13.31%</td>
+</tr>
+
+<tr>
+<td>I don’t know</td>
+<td>5</td>
+<td>1.62%</td>
+</tr>
+</tbody>
+</table>
+
+<p>The results are rather quite self-explaining. Improved performance of Airflow, better
+UI, and more telemetry are desirable. But this should go in pair with improved
+documentation and resources about using the Airflow, especially when we
+take into account the problem of onboarding new users.</p>
+
+<p>Another interesting point from that question is that only 16% think that operators
+should be extended and improved. This suggests that we should focus on improving
+Airflow core instead of adding more and more integrations.</p>
+
+<p><strong>What would be the most interesting feature for you?</strong></p>
+
+<table>
+<thead>
+<tr>
+<th></th>
+<th>No.</th>
+<th>%</th>
+</tr>
+</thead>
+
+<tbody>
+<tr>
+<td>Production-ready Airflow docker image</td>
+<td>175</td>
+<td>56.82%</td>
+</tr>
+
+<tr>
+<td>Declarative way of writing DAGs / automated DAGs generation</td>
+<td>155</td>
+<td>50.32%</td>
+</tr>
+
+<tr>
+<td>Horizontal Autoscaling</td>
+<td>122</td>
+<td>39.61%</td>
+</tr>
+
+<tr>
+<td>Asynchronous Operators</td>
+<td>97</td>
+<td>31.49%</td>
+</tr>
+
+<tr>
+<td>Stateless web server</td>
+<td>81</td>
+<td>26.3%</td>
+</tr>
+
+<tr>
+<td>Knative Executor</td>
+<td>48</td>
+<td>15.58%</td>
+</tr>
+
+<tr>
+<td>I already have all I need</td>
+<td>13</td>
+<td>4.22%</td>
+</tr>
+</tbody>
+</table>
+
+<p>Production Docker image wins, and it’s not a surprise. We all know that deploying
+Airflow is not a plug and play process, and that’s why the official image is being
+worked on by Jarek Potiuk. An unexpected result is that half of the users would like to
+have a declarative way of creating DAGs. That seems to be something that is “against Airflow”
+as we always emphasize the possibility of writing workflows in pure python. Stories
+about DAG generators are not new and confirm that there’s a need for a way to
+declare DAGs.</p>
+
+<h2 id="data">Data</h2>
+
+<p>If you think I missed something and you want to look for insights on your own the data is available
+for you here:</p>
+
+<ul>
+<li>Original data: <a href="https://storage.googleapis.com/airflow-survey/survey.csv" target="_blank">https://storage.googleapis.com/airflow-survey/survey.csv</a></li>
+<li>Processed: <a href="https://storage.googleapis.com/airflow-survey/airflow_survey_processed.csv" target="_blank">https://storage.googleapis.com/airflow-survey/airflow_survey_processed.csv</a></li>
+</ul>
+
+<p>The processed data includes multi-choice options one-hot encoded. If you find any interesting
+insight, please update the article (<a href="https://github.com/apache/airflow-site/blob/aip-11/CONTRIBUTE.md" target="_blank">make PR</a>
+to Airflow site).</p>
+
+ </description>
+ </item>
+
+ <item>
+ <title>Blog: New Airflow website</title>
+ <link>/blog/announcing-new-website/</link>
+ <pubDate>Wed, 11 Dec 2019 00:00:00 +0000</pubDate>
+
+ <guid>/blog/announcing-new-website/</guid>
+ <description>
+
+
+ <p>The brand <a href="https://airflow.apache.org/" target="_blank">new Airflow website</a> has arrived! Those who have been following the process know that the journey to update <a href="https://airflow.readthedocs.io/en/1.10.6/" target="_blank">the old Airflow website</a> started at the beginning of the year.
+Thanks to sponsorship from the Cloud Composer team at Google that allowed us to
+collaborate with <a href="https://www.polidea.com/" target="_blank">Polidea</a> and with their design studio <a href="https://utilodesign.com/" target="_blank">Utilo</a>, and deliver an awesome website.</p>
+
+<p>Documentation of open source projects is key to engaging new contributors in the maintenance,
+development, and adoption of software. We want the Apache Airflow community to have
+the best possible experience to contribute and use the project. We also took this opportunity to make the project
+more accessible, and in doing so, increase its reach.</p>
+
+<p>In the past three and a half months, we have updated everything: created a more efficient landing page,
+enhanced information architecture, and improved UX &amp; UI. Most importantly, the website now has capabilities
+to be translated into many languages. This is our effort to foster a more inclusive community around
+Apache Airflow, and we look forward to seeing contributions in Spanish, Chinese, Russian, and other languages as well!</p>
+
+<p>We built our website on Docsy, a platform that is easy to use and contribute to. Follow
+<a href="https://github.com/apache/airflow-site/blob/aip-11/README.md" target="_blank">these steps</a> to set up your environment and
+to create your first pull request. You may also use
+the new website for your own open source project as a template.
+All of our <a href="https://github.com/apache/airflow-site/tree/aip-11" target="_blank">code is open and hosted on Github</a>.</p>
+
+<p>Share your questions, comments, and suggestions with us, to help us improve the website.
+We hope that this new design makes finding documentation about Airflow easier,
+and that its improved accessibility increases adoption and use of Apache Airflow around the world.</p>
+
+<p>Happy browsing!</p>
+
+ </description>
+ </item>
+
+ <item>
<title>Blog: ApacheCon Europe 2019 — Thoughts and Insights by Airflow Commiters</title>
<link>/blog/apache-con-europe-2019-thoughts-and-insights-by-airflow-commiters/</link>
<pubDate>Fri, 22 Nov 2019 00:00:00 +0000</pubDate>
diff --git a/install/index.html b/install/index.html
index 2d93364..f259ff3 100644
--- a/install/index.html
+++ b/install/index.html
@@ -77,15 +77,15 @@ if (!doNotTrack) {
- <link rel="preload" href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" as="style">
- <link href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" rel="stylesheet" integrity="">
+ <link rel="preload" href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" as="style">
+ <link href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" rel="stylesheet" integrity="">
-<link rel="preload" href="/main.b2992.js" as="script">
-<link rel="preload" href="/chunk-1.b2992.js" as="script">
+<link rel="preload" href="/main.e3a02.js" as="script">
+<link rel="preload" href="/chunk-1.e3a02.js" as="script">
-<link rel="preload" href="/chunk-4.b2992.js" as="script">
+<link rel="preload" href="/chunk-4.e3a02.js" as="script">
@@ -770,6 +770,6 @@ if (!doNotTrack) {
</body>
- <script src="/main.b2992.js"></script>
+ <script src="/main.e3a02.js"></script>
</html>
diff --git a/main.e3a02.js b/main.e3a02.js
new file mode 100644
index 0000000..9329f12
--- /dev/null
+++ b/main.e3a02.js
@@ -0,0 +1 @@
+!function(c){function t(t){for(var e,n,r=t[0],o=t[1],i=0,a=[];i<r.length;i++)n=r[i],Object.prototype.hasOwnProperty.call(u,n)&&u[n]&&a.push(u[n][0]),u[n]=0;for(e in o)Object.prototype.hasOwnProperty.call(o,e)&&(c[e]=o[e]);for(l&&l(t);a.length;)a.shift()()}var n={},u={2:0};function s(t){if(n[t])return n[t].exports;var e=n[t]={i:t,l:!1,exports:{}};return c[t].call(e.exports,e,e.exports,s),e.l=!0,e.exports}s.e=function(o){var t=[],n=u[o];if(0!==n)if(n)t.push(n[2]);else{var e=new Promise(fun [...]
\ No newline at end of file
diff --git a/meetups/index.html b/meetups/index.html
index 77054c8..141bb68 100644
--- a/meetups/index.html
+++ b/meetups/index.html
@@ -77,15 +77,15 @@ if (!doNotTrack) {
- <link rel="preload" href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" as="style">
- <link href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" rel="stylesheet" integrity="">
+ <link rel="preload" href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" as="style">
+ <link href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" rel="stylesheet" integrity="">
-<link rel="preload" href="/main.b2992.js" as="script">
-<link rel="preload" href="/chunk-1.b2992.js" as="script">
+<link rel="preload" href="/main.e3a02.js" as="script">
+<link rel="preload" href="/chunk-1.e3a02.js" as="script">
-<link rel="preload" href="/chunk-4.b2992.js" as="script">
+<link rel="preload" href="/chunk-4.e3a02.js" as="script">
@@ -983,6 +983,6 @@ if (!doNotTrack) {
</body>
- <script src="/main.b2992.js"></script>
+ <script src="/main.e3a02.js"></script>
</html>
diff --git a/privacy-notice/index.html b/privacy-notice/index.html
index 2245155..83285e7 100644
--- a/privacy-notice/index.html
+++ b/privacy-notice/index.html
@@ -77,15 +77,15 @@ if (!doNotTrack) {
- <link rel="preload" href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" as="style">
- <link href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" rel="stylesheet" integrity="">
+ <link rel="preload" href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" as="style">
+ <link href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" rel="stylesheet" integrity="">
-<link rel="preload" href="/main.b2992.js" as="script">
-<link rel="preload" href="/chunk-1.b2992.js" as="script">
+<link rel="preload" href="/main.e3a02.js" as="script">
+<link rel="preload" href="/chunk-1.e3a02.js" as="script">
-<link rel="preload" href="/chunk-4.b2992.js" as="script">
+<link rel="preload" href="/chunk-4.e3a02.js" as="script">
@@ -604,6 +604,6 @@ if (!doNotTrack) {
</body>
- <script src="/main.b2992.js"></script>
+ <script src="/main.e3a02.js"></script>
</html>
diff --git a/roadmap/index.html b/roadmap/index.html
index e5ed110..9af7152 100644
--- a/roadmap/index.html
+++ b/roadmap/index.html
@@ -77,15 +77,15 @@ if (!doNotTrack) {
- <link rel="preload" href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" as="style">
- <link href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" rel="stylesheet" integrity="">
+ <link rel="preload" href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" as="style">
+ <link href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" rel="stylesheet" integrity="">
-<link rel="preload" href="/main.b2992.js" as="script">
-<link rel="preload" href="/chunk-1.b2992.js" as="script">
+<link rel="preload" href="/main.e3a02.js" as="script">
+<link rel="preload" href="/chunk-1.e3a02.js" as="script">
-<link rel="preload" href="/chunk-4.b2992.js" as="script">
+<link rel="preload" href="/chunk-4.e3a02.js" as="script">
@@ -774,6 +774,6 @@ not minutes) since the scheduler will execute it periodically to reflect the cha
</body>
- <script src="/main.b2992.js"></script>
+ <script src="/main.e3a02.js"></script>
</html>
diff --git a/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css b/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css
new file mode 100644
index 0000000..7eddbb5
--- /dev/null
+++ b/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css
@@ -0,0 +1 @@
+@charset "UTF-8";@import "https://fonts.googleapis.com/css?family=Rubik:500&display=swap";@import "https://fonts.googleapis.com/css?family=Roboto:400,400i,500,700&display=swap";@import "https://fonts.googleapis.com/css?family=Roboto+Mono:400,700&display=swap";.header__large--cerulean-blue{font-family:rubik,sans-serif;font-weight:500;font-size:72px;line-height:1.17;color:#017cee}.header__large--shamrock{font-family:rubik,sans-serif;font-weight:500;font-size:72px;line-height:1.17;color:#00 [...]
\ No newline at end of file
diff --git a/search/index.html b/search/index.html
index 2ba318f..a7c453c 100644
--- a/search/index.html
+++ b/search/index.html
@@ -83,15 +83,15 @@ if (!doNotTrack) {
- <link rel="preload" href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" as="style">
- <link href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" rel="stylesheet" integrity="">
+ <link rel="preload" href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" as="style">
+ <link href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" rel="stylesheet" integrity="">
-<link rel="preload" href="/main.b2992.js" as="script">
-<link rel="preload" href="/chunk-1.b2992.js" as="script">
+<link rel="preload" href="/main.e3a02.js" as="script">
+<link rel="preload" href="/chunk-1.e3a02.js" as="script">
-<link rel="preload" href="/chunk-4.b2992.js" as="script">
+<link rel="preload" href="/chunk-4.e3a02.js" as="script">
@@ -560,6 +560,6 @@ if (!doNotTrack) {
</body>
- <script src="/main.b2992.js"></script>
+ <script src="/main.e3a02.js"></script>
</html>
diff --git a/sitemap.xml b/sitemap.xml
index 2f961e3..41f3086 100644
--- a/sitemap.xml
+++ b/sitemap.xml
@@ -83,27 +83,47 @@
</url>
<url>
- <loc>/blog/apache-con-europe-2019-thoughts-and-insights-by-airflow-commiters/</loc>
- <lastmod>2019-11-26T12:05:52+01:00</lastmod>
+ <loc>/blog/airflow-survey/</loc>
+ <lastmod>2019-12-18T17:24:41+01:00</lastmod>
</url>
<url>
<loc>/blog/tags/community/</loc>
- <lastmod>2019-11-26T12:05:52+01:00</lastmod>
+ <lastmod>2019-12-18T17:24:41+01:00</lastmod>
</url>
<url>
- <loc>/blog/tags/development/</loc>
+ <loc>/blog/announcing-new-website/</loc>
+ <lastmod>2019-12-18T08:19:54-08:00</lastmod>
+ </url>
+
+ <url>
+ <loc>/blog/tags/survey/</loc>
+ <lastmod>2019-12-18T17:24:41+01:00</lastmod>
+ </url>
+
+ <url>
+ <loc>/tags/</loc>
+ <lastmod>2019-12-18T17:24:41+01:00</lastmod>
+ </url>
+
+ <url>
+ <loc>/blog/tags/users/</loc>
+ <lastmod>2019-12-18T17:24:41+01:00</lastmod>
+ </url>
+
+ <url>
+ <loc>/blog/apache-con-europe-2019-thoughts-and-insights-by-airflow-commiters/</loc>
<lastmod>2019-11-26T12:05:52+01:00</lastmod>
</url>
<url>
- <loc>/blog/its-a-breeze-to-develop-apache-airflow/</loc>
+ <loc>/blog/tags/development/</loc>
<lastmod>2019-11-26T12:05:52+01:00</lastmod>
</url>
<url>
- <loc>/tags/</loc>
+ <loc>/blog/its-a-breeze-to-develop-apache-airflow/</loc>
<lastmod>2019-11-26T12:05:52+01:00</lastmod>
</url>
@@ -124,7 +144,7 @@
<url>
<loc>/use-cases/adobe/</loc>
- <lastmod>2019-11-15T13:01:44+01:00</lastmod>
+ <lastmod>2019-11-29T14:27:18+11:00</lastmod>
</url>
<url>
diff --git a/tags/index.html b/tags/index.html
index d9061c3..1cae044 100644
--- a/tags/index.html
+++ b/tags/index.html
@@ -37,7 +37,7 @@
<meta property="og:image" content="/images/feature-image.png" />
-<meta property="og:updated_time" content="2019-11-22T00:00:00+00:00" /><meta property="og:site_name" content="Apache Airflow" />
+<meta property="og:updated_time" content="2019-12-11T00:00:00+00:00" /><meta property="og:site_name" content="Apache Airflow" />
<meta itemprop="name" content="Tags">
<meta itemprop="description" content="Platform created by community to programmatically author, schedule and monitor workflows.">
@@ -78,15 +78,15 @@ if (!doNotTrack) {
- <link rel="preload" href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" as="style">
- <link href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" rel="stylesheet" integrity="">
+ <link rel="preload" href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" as="style">
+ <link href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" rel="stylesheet" integrity="">
-<link rel="preload" href="/main.b2992.js" as="script">
-<link rel="preload" href="/chunk-1.b2992.js" as="script">
+<link rel="preload" href="/main.e3a02.js" as="script">
+<link rel="preload" href="/chunk-1.e3a02.js" as="script">
-<link rel="preload" href="/chunk-4.b2992.js" as="script">
+<link rel="preload" href="/chunk-4.e3a02.js" as="script">
@@ -549,6 +549,6 @@ if (!doNotTrack) {
</body>
- <script src="/main.b2992.js"></script>
+ <script src="/main.e3a02.js"></script>
</html>
diff --git a/tags/index.xml b/tags/index.xml
index cafc622..2d76659 100644
--- a/tags/index.xml
+++ b/tags/index.xml
@@ -4,7 +4,7 @@
<link>/tags/</link>
<description>Recent content in Tags on Apache Airflow</description>
<generator>Hugo -- gohugo.io</generator>
- <lastBuildDate>Fri, 22 Nov 2019 00:00:00 +0000</lastBuildDate>
+ <lastBuildDate>Wed, 11 Dec 2019 00:00:00 +0000</lastBuildDate>
<atom:link href="/tags/index.xml" rel="self" type="application/rss+xml" />
diff --git a/use-cases/adobe/index.html b/use-cases/adobe/index.html
index 22a412f..7af9ea5 100644
--- a/use-cases/adobe/index.html
+++ b/use-cases/adobe/index.html
@@ -35,12 +35,12 @@
<meta property="og:url" content="/use-cases/adobe/" />
<meta property="og:image" content="/images/feature-image.png" />
-<meta property="article:modified_time" content="2019-11-15T13:01:44+01:00" /><meta property="og:site_name" content="Apache Airflow" />
+<meta property="article:modified_time" content="2019-11-29T14:27:18+11:00" /><meta property="og:site_name" content="Apache Airflow" />
<meta itemprop="name" content="Adobe">
<meta itemprop="description" content="What was the problem? Modern big data platforms need sophisticated data pipelines connecting to many backend services enabling complex workflows. These workflows need to be deployed, monitored, and run either on regular schedules or triggered by external events. Adobe Experience Platform component services architected and built an orchestration service to enable their users to author, schedule, and monitor complex hierarchical (including sequential a [...]
-<meta itemprop="dateModified" content="2019-11-15T13:01:44+01:00" />
+<meta itemprop="dateModified" content="2019-11-29T14:27:18+11:00" />
<meta itemprop="wordCount" content="251">
@@ -83,15 +83,15 @@ if (!doNotTrack) {
- <link rel="preload" href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" as="style">
- <link href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" rel="stylesheet" integrity="">
+ <link rel="preload" href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" as="style">
+ <link href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" rel="stylesheet" integrity="">
-<link rel="preload" href="/main.b2992.js" as="script">
-<link rel="preload" href="/chunk-1.b2992.js" as="script">
+<link rel="preload" href="/main.e3a02.js" as="script">
+<link rel="preload" href="/chunk-1.e3a02.js" as="script">
-<link rel="preload" href="/chunk-4.b2992.js" as="script">
+<link rel="preload" href="/chunk-4.e3a02.js" as="script">
@@ -377,7 +377,7 @@ if (!doNotTrack) {
<div class="quote">
- <p class="quote--text">Apache Aiflow is highly extensible and its plugin interface can be used to meet a variety of use cases. It supports variety of deployment models and has a very active community to scale innovation.</p>
+ <p class="quote--text">Apache Airflow is highly extensible and its plugin interface can be used to meet a variety of use cases. It supports variety of deployment models and has a very active community to scale innovation.</p>
<p class="quote--author">Raman Gupta</p>
<img src="/usecase-logos/adobe-logo.svg" alt="Adobe logo" class="quote--logo" />
@@ -605,6 +605,6 @@ if (!doNotTrack) {
</body>
- <script src="/main.b2992.js"></script>
+ <script src="/main.e3a02.js"></script>
</html>
diff --git a/use-cases/big-fish-games/index.html b/use-cases/big-fish-games/index.html
index d02e666..04985c1 100644
--- a/use-cases/big-fish-games/index.html
+++ b/use-cases/big-fish-games/index.html
@@ -83,15 +83,15 @@ if (!doNotTrack) {
- <link rel="preload" href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" as="style">
- <link href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" rel="stylesheet" integrity="">
+ <link rel="preload" href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" as="style">
+ <link href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" rel="stylesheet" integrity="">
-<link rel="preload" href="/main.b2992.js" as="script">
-<link rel="preload" href="/chunk-1.b2992.js" as="script">
+<link rel="preload" href="/main.e3a02.js" as="script">
+<link rel="preload" href="/chunk-1.e3a02.js" as="script">
-<link rel="preload" href="/chunk-4.b2992.js" as="script">
+<link rel="preload" href="/chunk-4.e3a02.js" as="script">
@@ -605,6 +605,6 @@ if (!doNotTrack) {
</body>
- <script src="/main.b2992.js"></script>
+ <script src="/main.e3a02.js"></script>
</html>
diff --git a/use-cases/dish/index.html b/use-cases/dish/index.html
index a3ae871..039889a 100644
--- a/use-cases/dish/index.html
+++ b/use-cases/dish/index.html
@@ -83,15 +83,15 @@ if (!doNotTrack) {
- <link rel="preload" href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" as="style">
- <link href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" rel="stylesheet" integrity="">
+ <link rel="preload" href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" as="style">
+ <link href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" rel="stylesheet" integrity="">
-<link rel="preload" href="/main.b2992.js" as="script">
-<link rel="preload" href="/chunk-1.b2992.js" as="script">
+<link rel="preload" href="/main.e3a02.js" as="script">
+<link rel="preload" href="/chunk-1.e3a02.js" as="script">
-<link rel="preload" href="/chunk-4.b2992.js" as="script">
+<link rel="preload" href="/chunk-4.e3a02.js" as="script">
@@ -605,6 +605,6 @@ if (!doNotTrack) {
</body>
- <script src="/main.b2992.js"></script>
+ <script src="/main.e3a02.js"></script>
</html>
diff --git a/use-cases/experity/index.html b/use-cases/experity/index.html
index 4116fe2..8b08263 100644
--- a/use-cases/experity/index.html
+++ b/use-cases/experity/index.html
@@ -86,15 +86,15 @@ if (!doNotTrack) {
- <link rel="preload" href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" as="style">
- <link href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" rel="stylesheet" integrity="">
+ <link rel="preload" href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" as="style">
+ <link href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" rel="stylesheet" integrity="">
-<link rel="preload" href="/main.b2992.js" as="script">
-<link rel="preload" href="/chunk-1.b2992.js" as="script">
+<link rel="preload" href="/main.e3a02.js" as="script">
+<link rel="preload" href="/chunk-1.e3a02.js" as="script">
-<link rel="preload" href="/chunk-4.b2992.js" as="script">
+<link rel="preload" href="/chunk-4.e3a02.js" as="script">
@@ -608,6 +608,6 @@ if (!doNotTrack) {
</body>
- <script src="/main.b2992.js"></script>
+ <script src="/main.e3a02.js"></script>
</html>
diff --git a/use-cases/index.html b/use-cases/index.html
index dd20bb6..912d6b5 100644
--- a/use-cases/index.html
+++ b/use-cases/index.html
@@ -77,15 +77,15 @@ if (!doNotTrack) {
- <link rel="preload" href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" as="style">
- <link href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" rel="stylesheet" integrity="">
+ <link rel="preload" href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" as="style">
+ <link href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" rel="stylesheet" integrity="">
-<link rel="preload" href="/main.b2992.js" as="script">
-<link rel="preload" href="/chunk-1.b2992.js" as="script">
+<link rel="preload" href="/main.e3a02.js" as="script">
+<link rel="preload" href="/chunk-1.e3a02.js" as="script">
-<link rel="preload" href="/chunk-4.b2992.js" as="script">
+<link rel="preload" href="/chunk-4.e3a02.js" as="script">
@@ -382,7 +382,7 @@ if (!doNotTrack) {
</div>
<p class="box-event__case-study--quote"
- >Apache Aiflow is highly extensible and its plugin interface can be used to meet a variety of use cases. It supports …</p>
+ >Apache Airflow is highly extensible and its plugin interface can be used to meet a variety of use cases. It supports …</p>
<button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" >Learn more</button>
@@ -672,6 +672,6 @@ if (!doNotTrack) {
</body>
- <script src="/main.b2992.js"></script>
+ <script src="/main.e3a02.js"></script>
</html>
diff --git a/use-cases/onefootball/index.html b/use-cases/onefootball/index.html
index 36f5c68..ef876ed 100644
--- a/use-cases/onefootball/index.html
+++ b/use-cases/onefootball/index.html
@@ -86,15 +86,15 @@ if (!doNotTrack) {
- <link rel="preload" href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" as="style">
- <link href="/scss/main-custom.min.fee6b31b7089d24aaebfd0b9b208df88b87fb0bd3d9607f0a91c2c91d71bc682.css" rel="stylesheet" integrity="">
+ <link rel="preload" href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" as="style">
+ <link href="/scss/main-custom.min.31b9df8bbe04b2191429437660cec2f3e9245a57bf13bc82c8c4d129a9b1f00e.css" rel="stylesheet" integrity="">
-<link rel="preload" href="/main.b2992.js" as="script">
-<link rel="preload" href="/chunk-1.b2992.js" as="script">
+<link rel="preload" href="/main.e3a02.js" as="script">
+<link rel="preload" href="/chunk-1.e3a02.js" as="script">
-<link rel="preload" href="/chunk-4.b2992.js" as="script">
+<link rel="preload" href="/chunk-4.e3a02.js" as="script">
@@ -612,6 +612,6 @@ if (!doNotTrack) {
</body>
- <script src="/main.b2992.js"></script>
+ <script src="/main.e3a02.js"></script>
</html>