You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by fo...@apache.org on 2018/03/19 21:02:18 UTC
incubator-airflow git commit: [AIRFLOW-2226] Rename
google_cloud_storage_default to google_cloud_default
Repository: incubator-airflow
Updated Branches:
refs/heads/master e83b01551 -> bb287ecf5
[AIRFLOW-2226] Rename google_cloud_storage_default to google_cloud_default
The Google cloud operators uses both
google_cloud_storage_default and
google_cloud_default as a default conn_id. This is
confusing and the
google_cloud_storage_default conn_id isnt
initialized by default in db.py
Therefore we rename the
google_cloud_storage_default to
google_cloud_default for simplicity and
convenience
Closes #3141 from Fokko/airflow-2226
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/bb287ecf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/bb287ecf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/bb287ecf
Branch: refs/heads/master
Commit: bb287ecf5bd1777124801c14750c6b685cc27c76
Parents: e83b015
Author: Fokko Driesprong <fo...@godatadriven.com>
Authored: Mon Mar 19 22:02:10 2018 +0100
Committer: Fokko Driesprong <fo...@godatadriven.com>
Committed: Mon Mar 19 22:02:10 2018 +0100
----------------------------------------------------------------------
UPDATING.md | 4 ++++
airflow/contrib/hooks/gcs_hook.py | 2 +-
airflow/contrib/operators/bigquery_operator.py | 4 ++--
airflow/contrib/operators/file_to_gcs.py | 4 ++--
airflow/contrib/operators/gcs_copy_operator.py | 2 +-
airflow/contrib/operators/gcs_download_operator.py | 2 +-
airflow/contrib/operators/gcs_list_operator.py | 2 +-
airflow/contrib/operators/gcs_operator.py | 2 +-
airflow/contrib/operators/gcs_to_bq.py | 2 +-
airflow/contrib/operators/gcs_to_gcs.py | 2 +-
airflow/contrib/operators/mysql_to_gcs.py | 2 +-
airflow/contrib/operators/postgres_to_gcs_operator.py | 2 +-
airflow/contrib/sensors/gcs_sensor.py | 6 +++---
13 files changed, 20 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/bb287ecf/UPDATING.md
----------------------------------------------------------------------
diff --git a/UPDATING.md b/UPDATING.md
index e03243f..21581bf 100644
--- a/UPDATING.md
+++ b/UPDATING.md
@@ -24,6 +24,10 @@ Dataflow job labeling is now supported in Dataflow{Java,Python}Operator with a d
"airflow-version" label, please upgrade your google-cloud-dataflow or apache-beam version
to 2.2.0 or greater.
+### Google cloud connection string
+
+With Airflow 1.9 or lower there where two connection strings for the Google Cloud operators, both `google_cloud_storage_default` and `google_cloud_default`. This can be confusing and therefore the `google_cloud_storage_default` connection id has been replaced with `google_cloud_default` to make the connection id consistent across Airflow.
+
## Airflow 1.9
### SSH Hook updates, along with new SSH Operator & SFTP Operator
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/bb287ecf/airflow/contrib/hooks/gcs_hook.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/hooks/gcs_hook.py b/airflow/contrib/hooks/gcs_hook.py
index 8c1e7bb..6a53b1d 100644
--- a/airflow/contrib/hooks/gcs_hook.py
+++ b/airflow/contrib/hooks/gcs_hook.py
@@ -27,7 +27,7 @@ class GoogleCloudStorageHook(GoogleCloudBaseHook):
"""
def __init__(self,
- google_cloud_storage_conn_id='google_cloud_storage_default',
+ google_cloud_storage_conn_id='google_cloud_default',
delegate_to=None):
super(GoogleCloudStorageHook, self).__init__(google_cloud_storage_conn_id,
delegate_to)
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/bb287ecf/airflow/contrib/operators/bigquery_operator.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/operators/bigquery_operator.py b/airflow/contrib/operators/bigquery_operator.py
index ef1ada0..6f1cc6c 100644
--- a/airflow/contrib/operators/bigquery_operator.py
+++ b/airflow/contrib/operators/bigquery_operator.py
@@ -242,7 +242,7 @@ class BigQueryCreateEmptyTableOperator(BaseOperator):
gcs_schema_object=None,
time_partitioning={},
bigquery_conn_id='bigquery_default',
- google_cloud_storage_conn_id='google_cloud_storage_default',
+ google_cloud_storage_conn_id='google_cloud_default',
delegate_to=None,
*args, **kwargs):
@@ -376,7 +376,7 @@ class BigQueryCreateExternalTableOperator(BaseOperator):
allow_quoted_newlines=False,
allow_jagged_rows=False,
bigquery_conn_id='bigquery_default',
- google_cloud_storage_conn_id='google_cloud_storage_default',
+ google_cloud_storage_conn_id='google_cloud_default',
delegate_to=None,
src_fmt_configs={},
*args, **kwargs):
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/bb287ecf/airflow/contrib/operators/file_to_gcs.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/operators/file_to_gcs.py b/airflow/contrib/operators/file_to_gcs.py
index b9f68f9..eaf8c9d 100644
--- a/airflow/contrib/operators/file_to_gcs.py
+++ b/airflow/contrib/operators/file_to_gcs.py
@@ -36,13 +36,13 @@ class FileToGoogleCloudStorageOperator(BaseOperator):
:type delegate_to: string
"""
template_fields = ('src', 'dst', 'bucket')
-
+
@apply_defaults
def __init__(self,
src,
dst,
bucket,
- google_cloud_storage_conn_id='google_cloud_storage_default',
+ google_cloud_storage_conn_id='google_cloud_default',
mime_type='application/octet-stream',
delegate_to=None,
*args,
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/bb287ecf/airflow/contrib/operators/gcs_copy_operator.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/operators/gcs_copy_operator.py b/airflow/contrib/operators/gcs_copy_operator.py
index 4922708..842d6dc 100644
--- a/airflow/contrib/operators/gcs_copy_operator.py
+++ b/airflow/contrib/operators/gcs_copy_operator.py
@@ -67,7 +67,7 @@ class GoogleCloudStorageCopyOperator(BaseOperator):
source_files_delimiter=None,
destination_bucket=None,
destination_directory='',
- google_cloud_storage_conn_id='google_cloud_storage_default',
+ google_cloud_storage_conn_id='google_cloud_default',
delegate_to=None,
*args,
**kwargs):
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/bb287ecf/airflow/contrib/operators/gcs_download_operator.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/operators/gcs_download_operator.py b/airflow/contrib/operators/gcs_download_operator.py
index 75de608..f6276b5 100644
--- a/airflow/contrib/operators/gcs_download_operator.py
+++ b/airflow/contrib/operators/gcs_download_operator.py
@@ -53,7 +53,7 @@ class GoogleCloudStorageDownloadOperator(BaseOperator):
object,
filename=None,
store_to_xcom_key=None,
- google_cloud_storage_conn_id='google_cloud_storage_default',
+ google_cloud_storage_conn_id='google_cloud_default',
delegate_to=None,
*args,
**kwargs):
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/bb287ecf/airflow/contrib/operators/gcs_list_operator.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/operators/gcs_list_operator.py b/airflow/contrib/operators/gcs_list_operator.py
index 3c49cc1..a708524 100644
--- a/airflow/contrib/operators/gcs_list_operator.py
+++ b/airflow/contrib/operators/gcs_list_operator.py
@@ -60,7 +60,7 @@ class GoogleCloudStorageListOperator(BaseOperator):
bucket,
prefix=None,
delimiter=None,
- google_cloud_storage_conn_id='google_cloud_storage_default',
+ google_cloud_storage_conn_id='google_cloud_default',
delegate_to=None,
*args,
**kwargs):
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/bb287ecf/airflow/contrib/operators/gcs_operator.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/operators/gcs_operator.py b/airflow/contrib/operators/gcs_operator.py
index d92d125..f03b5ec 100644
--- a/airflow/contrib/operators/gcs_operator.py
+++ b/airflow/contrib/operators/gcs_operator.py
@@ -85,7 +85,7 @@ class GoogleCloudStorageCreateBucketOperator(BaseOperator):
location='US',
project_id=None,
labels=None,
- google_cloud_storage_conn_id='google_cloud_storage_default',
+ google_cloud_storage_conn_id='google_cloud_default',
delegate_to=None,
*args,
**kwargs):
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/bb287ecf/airflow/contrib/operators/gcs_to_bq.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/operators/gcs_to_bq.py b/airflow/contrib/operators/gcs_to_bq.py
index be15e52..3126ae0 100644
--- a/airflow/contrib/operators/gcs_to_bq.py
+++ b/airflow/contrib/operators/gcs_to_bq.py
@@ -135,7 +135,7 @@ class GoogleCloudStorageToBigQueryOperator(BaseOperator):
allow_jagged_rows=False,
max_id_key=None,
bigquery_conn_id='bigquery_default',
- google_cloud_storage_conn_id='google_cloud_storage_default',
+ google_cloud_storage_conn_id='google_cloud_default',
delegate_to=None,
schema_update_options=(),
src_fmt_configs={},
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/bb287ecf/airflow/contrib/operators/gcs_to_gcs.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/operators/gcs_to_gcs.py b/airflow/contrib/operators/gcs_to_gcs.py
index 8db3067..b6e49dc 100644
--- a/airflow/contrib/operators/gcs_to_gcs.py
+++ b/airflow/contrib/operators/gcs_to_gcs.py
@@ -65,7 +65,7 @@ class GoogleCloudStorageToGoogleCloudStorageOperator(BaseOperator):
destination_bucket=None,
destination_object=None,
move_object=False,
- google_cloud_storage_conn_id='google_cloud_storage_default',
+ google_cloud_storage_conn_id='google_cloud_default',
delegate_to=None,
*args,
**kwargs):
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/bb287ecf/airflow/contrib/operators/mysql_to_gcs.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/operators/mysql_to_gcs.py b/airflow/contrib/operators/mysql_to_gcs.py
index 9ba84c7..2249b9c 100644
--- a/airflow/contrib/operators/mysql_to_gcs.py
+++ b/airflow/contrib/operators/mysql_to_gcs.py
@@ -45,7 +45,7 @@ class MySqlToGoogleCloudStorageOperator(BaseOperator):
schema_filename=None,
approx_max_file_size_bytes=1900000000,
mysql_conn_id='mysql_default',
- google_cloud_storage_conn_id='google_cloud_storage_default',
+ google_cloud_storage_conn_id='google_cloud_default',
schema=None,
delegate_to=None,
*args,
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/bb287ecf/airflow/contrib/operators/postgres_to_gcs_operator.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/operators/postgres_to_gcs_operator.py b/airflow/contrib/operators/postgres_to_gcs_operator.py
index ab6fdf4..8224b36 100644
--- a/airflow/contrib/operators/postgres_to_gcs_operator.py
+++ b/airflow/contrib/operators/postgres_to_gcs_operator.py
@@ -44,7 +44,7 @@ class PostgresToGoogleCloudStorageOperator(BaseOperator):
schema_filename=None,
approx_max_file_size_bytes=1900000000,
postgres_conn_id='postgres_default',
- google_cloud_storage_conn_id='google_cloud_storage_default',
+ google_cloud_storage_conn_id='google_cloud_default',
delegate_to=None,
parameters=None,
*args,
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/bb287ecf/airflow/contrib/sensors/gcs_sensor.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/sensors/gcs_sensor.py b/airflow/contrib/sensors/gcs_sensor.py
index d91e2b1..aa7afd3 100644
--- a/airflow/contrib/sensors/gcs_sensor.py
+++ b/airflow/contrib/sensors/gcs_sensor.py
@@ -28,7 +28,7 @@ class GoogleCloudStorageObjectSensor(BaseSensorOperator):
self,
bucket,
object, # pylint:disable=redefined-builtin
- google_cloud_conn_id='google_cloud_storage_default',
+ google_cloud_conn_id='google_cloud_default',
delegate_to=None,
*args,
**kwargs):
@@ -84,7 +84,7 @@ class GoogleCloudStorageObjectUpdatedSensor(BaseSensorOperator):
bucket,
object, # pylint:disable=redefined-builtin
ts_func=ts_function,
- google_cloud_conn_id='google_cloud_storage_default',
+ google_cloud_conn_id='google_cloud_default',
delegate_to=None,
*args,
**kwargs):
@@ -135,7 +135,7 @@ class GoogleCloudStoragePrefixSensor(BaseSensorOperator):
self,
bucket,
prefix,
- google_cloud_conn_id='google_cloud_storage_default',
+ google_cloud_conn_id='google_cloud_default',
delegate_to=None,
*args,
**kwargs):