You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by cr...@apache.org on 2017/07/05 17:53:00 UTC
incubator-airflow git commit: [AIRFLOW-1343] Fix dataproc label format
Repository: incubator-airflow
Updated Branches:
refs/heads/master f1d72e53b -> 644f5d43a
[AIRFLOW-1343] Fix dataproc label format
Dataproc label must conform to the following regex:
[a-z]([-a-z0-9]*[a-z0-9])?. Current "airflow_version"
label violates this format. This commit fixes the format
and updates the unittest to prevent future violations.
Closes #2413 from fenglu-g/master
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/644f5d43
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/644f5d43
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/644f5d43
Branch: refs/heads/master
Commit: 644f5d43a49d621dbd2e3f4d4731bf85c240f080
Parents: f1d72e5
Author: Feng Lu <fe...@fengcloud.hot.corp.google.com>
Authored: Wed Jul 5 10:52:20 2017 -0700
Committer: Chris Riccomini <cr...@apache.org>
Committed: Wed Jul 5 10:52:30 2017 -0700
----------------------------------------------------------------------
airflow/contrib/operators/dataproc_operator.py | 6 +++++-
tests/contrib/operators/test_dataproc_operator.py | 12 ++++++++----
2 files changed, 13 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/644f5d43/airflow/contrib/operators/dataproc_operator.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/operators/dataproc_operator.py b/airflow/contrib/operators/dataproc_operator.py
index 3e006ac..4b6b61c 100644
--- a/airflow/contrib/operators/dataproc_operator.py
+++ b/airflow/contrib/operators/dataproc_operator.py
@@ -229,7 +229,11 @@ class DataprocClusterCreateOperator(BaseOperator):
}
cluster_data['labels'] = self.labels if self.labels else {}
- cluster_data['labels'].update({'airflow_version': version})
+ # Dataproc labels must conform to the following regex:
+ # [a-z]([-a-z0-9]*[a-z0-9])? (current airflow version string follows
+ # semantic versioning spec: x.y.z).
+ cluster_data['labels'].update({'airflow-version':
+ 'v' + version.replace('.', '-')})
if self.storage_bucket:
cluster_data['config']['configBucket'] = self.storage_bucket
if self.metadata:
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/644f5d43/tests/contrib/operators/test_dataproc_operator.py
----------------------------------------------------------------------
diff --git a/tests/contrib/operators/test_dataproc_operator.py b/tests/contrib/operators/test_dataproc_operator.py
index a441e47..923fecc 100644
--- a/tests/contrib/operators/test_dataproc_operator.py
+++ b/tests/contrib/operators/test_dataproc_operator.py
@@ -13,6 +13,7 @@
# limitations under the License.
#
+import re
import unittest
from airflow.contrib.operators.dataproc_operator import DataprocClusterCreateOperator
@@ -33,12 +34,12 @@ WORKER_MACHINE_TYPE = 'n1-standard-2'
WORKER_DISK_SIZE = 100
NUM_PREEMPTIBLE_WORKERS = 2
LABEL1 = {}
-LABEL2 = {'application':'test', 'year': 2017}
+LABEL2 = {'application':'test', 'year': 2017}
class DataprocClusterCreateOperatorTest(unittest.TestCase):
# Unitest for the DataprocClusterCreateOperator
def setUp(self):
- # instantiate two different test cases with different labels
+ # instantiate two different test cases with different labels.
self.labels = [LABEL1, LABEL2]
self.dataproc_operators = []
for labels in self.labels:
@@ -86,8 +87,11 @@ class DataprocClusterCreateOperatorTest(unittest.TestCase):
self.assertEqual(cluster_data['config']['workerConfig']['numInstances'], NUM_WORKERS)
self.assertEqual(cluster_data['config']['secondaryWorkerConfig']['numInstances'],
NUM_PREEMPTIBLE_WORKERS)
- # test whether the default airflow_version label has been properly set to the dataproc operator
+ # test whether the default airflow-version label has been properly
+ # set to the dataproc operator.
merged_labels = {}
merged_labels.update(self.labels[suffix])
- merged_labels.update({'airflow_version': version})
+ merged_labels.update({'airflow-version': 'v' + version.replace('.', '-')})
+ self.assertTrue(re.match(r'[a-z]([-a-z0-9]*[a-z0-9])?',
+ cluster_data['labels']['airflow-version']))
self.assertEqual(cluster_data['labels'], merged_labels)