You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by di...@apache.org on 2020/06/18 23:02:38 UTC
[airflow] branch v1-10-test updated: [AIRFLOW-4472] Use
json.dumps/loads for templating lineage data (#5253)
This is an automated email from the ASF dual-hosted git repository.
dimberman pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v1-10-test by this push:
new 132ea62 [AIRFLOW-4472] Use json.dumps/loads for templating lineage data (#5253)
132ea62 is described below
commit 132ea62137624ce5fafdd7e1d809fa5a115cf0c8
Author: bolkedebruin <bo...@users.noreply.github.com>
AuthorDate: Tue May 7 22:42:28 2019 +0200
[AIRFLOW-4472] Use json.dumps/loads for templating lineage data (#5253)
jinja2 cannot use dict/lists as templates hence converting
it to json solves this while keeping complexity down.
(cherry picked from commit a6daeb544e815fe350a96d24ae3bb14aee4079a7)
---
airflow/lineage/datasets.py | 11 +++++++++--
airflow/models/baseoperator.py | 7 ++++---
2 files changed, 13 insertions(+), 5 deletions(-)
diff --git a/airflow/lineage/datasets.py b/airflow/lineage/datasets.py
index 2602770..3d61e5d 100644
--- a/airflow/lineage/datasets.py
+++ b/airflow/lineage/datasets.py
@@ -16,6 +16,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+import json
import six
from typing import List
@@ -62,7 +63,11 @@ class DataSet(object):
if attr in self.attributes:
if self.context:
env = Environment()
- return env.from_string(self._data.get(attr)).render(**self.context)
+ # dump to json here in order to be able to manage dicts and lists
+ rendered = env.from_string(
+ json.dumps(self._data.get(attr))
+ ).render(**self.context)
+ return json.loads(rendered)
return self._data.get(attr)
@@ -82,7 +87,9 @@ class DataSet(object):
env = Environment()
if self.context:
for key, value in six.iteritems(attributes):
- attributes[key] = env.from_string(value).render(**self.context)
+ attributes[key] = json.loads(
+ env.from_string(json.dumps(value)).render(**self.context)
+ )
d = {
"typeName": self.type_name,
diff --git a/airflow/models/baseoperator.py b/airflow/models/baseoperator.py
index b61c980..470c14f 100644
--- a/airflow/models/baseoperator.py
+++ b/airflow/models/baseoperator.py
@@ -24,10 +24,11 @@ import functools
import logging
import sys
import warnings
+from datetime import timedelta, datetime
+from typing import Callable, Dict, Iterable, List, Optional, Set
from abc import ABCMeta, abstractmethod
from datetime import datetime, timedelta
-from typing import Any, Callable, ClassVar, Dict, FrozenSet, Iterable, List, Optional, Set, Type, Union
import attr
@@ -428,8 +429,8 @@ class BaseOperator(LoggingMixin):
self._log = logging.getLogger("airflow.task.operators")
# lineage
- self.inlets = [] # type: Iterable[DataSet]
- self.outlets = [] # type: Iterable[DataSet]
+ self.inlets = [] # type: List[DataSet]
+ self.outlets = [] # type: List[DataSet]
self.lineage_data = None
self._inlets = {