You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2021/07/26 08:43:55 UTC
[airflow] branch main updated: Fixes several failing tests after
broken main (#17222)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new d01cc94 Fixes several failing tests after broken main (#17222)
d01cc94 is described below
commit d01cc945ddb03620216159335729a36c1a20f9f2
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Mon Jul 26 10:43:41 2021 +0200
Fixes several failing tests after broken main (#17222)
Several problems slipped through after recent broken main event
end they were merged with failing tests. This PR fixes it as well
as brings correct approach for "initialize-local-virtualenv"
to get constraints from sources rather than PyPI constraints,
which makes it easier to synchronize with latest versions of
depenendencies.
---
airflow/providers/google/ads/hooks/ads.py | 2 +-
breeze | 4 ++--
tests/providers/amazon/aws/hooks/test_s3.py | 9 +++------
tests/providers/google/ads/hooks/test_ads.py | 14 +++++++++-----
tests/providers/google/cloud/hooks/test_bigquery.py | 10 +++++++++-
5 files changed, 24 insertions(+), 15 deletions(-)
diff --git a/airflow/providers/google/ads/hooks/ads.py b/airflow/providers/google/ads/hooks/ads.py
index 7222dac..35a17f6 100644
--- a/airflow/providers/google/ads/hooks/ads.py
+++ b/airflow/providers/google/ads/hooks/ads.py
@@ -239,7 +239,7 @@ class GoogleAdsHook(BaseHook):
request = self._get_client.get_type("SearchGoogleAdsRequest")
request.customer_id = client_id
request.query = query
- request.page_size = 10000
+ request.page_size = page_size
iterator = service.search(request=request)
iterators.append(iterator)
diff --git a/breeze b/breeze
index c991a53..60674b2 100755
--- a/breeze
+++ b/breeze
@@ -255,8 +255,8 @@ function breeze::initialize_virtualenv() {
echo
pushd "${AIRFLOW_SOURCES}" >/dev/null 2>&1 || exit 1
set +e
- pip install -e ".[devel]" \
- --constraint "https://raw.githubusercontent.com/${CONSTRAINTS_GITHUB_REPOSITORY}/${DEFAULT_CONSTRAINTS_BRANCH}/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt"
+ pip install -e ".[devel_all]" \
+ --constraint "https://raw.githubusercontent.com/${CONSTRAINTS_GITHUB_REPOSITORY}/${DEFAULT_CONSTRAINTS_BRANCH}/constraints-source-providers-${PYTHON_MAJOR_MINOR_VERSION}.txt"
res=$?
set -e
popd
diff --git a/tests/providers/amazon/aws/hooks/test_s3.py b/tests/providers/amazon/aws/hooks/test_s3.py
index ad91e4d..42d717d 100644
--- a/tests/providers/amazon/aws/hooks/test_s3.py
+++ b/tests/providers/amazon/aws/hooks/test_s3.py
@@ -26,7 +26,6 @@ import boto3
import pytest
from botocore.exceptions import ClientError, NoCredentialsError
-from airflow.exceptions import AirflowException
from airflow.models import Connection
from airflow.providers.amazon.aws.hooks.s3 import S3Hook, provide_bucket_name, unify_bucket_name_and_key
@@ -366,12 +365,10 @@ class TestAwsS3Hook:
assert test_bucket_name == 'bucket'
def test_delete_objects_key_does_not_exist(self, s3_bucket):
+ # The behaviour of delete changed in recent version of s3 mock libraries.
+ # It will succeed idempotently
hook = S3Hook()
- with pytest.raises(AirflowException) as ctx:
- hook.delete_objects(bucket=s3_bucket, keys=['key-1'])
-
- assert isinstance(ctx.value, AirflowException)
- assert str(ctx.value) == "Errors when deleting: ['key-1']"
+ hook.delete_objects(bucket=s3_bucket, keys=['key-1'])
def test_delete_objects_one_key(self, mocked_s3_res, s3_bucket):
key = 'key-1'
diff --git a/tests/providers/google/ads/hooks/test_ads.py b/tests/providers/google/ads/hooks/test_ads.py
index f5923ae..8d4ce01 100644
--- a/tests/providers/google/ads/hooks/test_ads.py
+++ b/tests/providers/google/ads/hooks/test_ads.py
@@ -16,6 +16,7 @@
# under the License.
from unittest import mock
+from unittest.mock import PropertyMock
import pytest
@@ -56,18 +57,21 @@ class TestGoogleAdsHook:
@mock.patch("airflow.providers.google.ads.hooks.ads.GoogleAdsClient")
def test_search(self, mock_client, mock_hook):
service = mock_client.load_from_dict.return_value.get_service.return_value
+ mock_client.load_from_dict.return_value.get_type.side_effect = [PropertyMock(), PropertyMock()]
+ client_ids = ["1", "2"]
rows = ["row1", "row2"]
service.search.side_effects = rows
# Here we mock _extract_rows to assert calls and
# avoid additional __iter__ calls
mock_hook._extract_rows = list
-
query = "QUERY"
- client_ids = ["1", "2"]
- mock_hook.search(client_ids=client_ids, query="QUERY", page_size=2)
- expected_calls = [mock.call(c, query=query, page_size=2) for c in client_ids]
- service.search.assert_has_calls(expected_calls)
+ mock_hook.search(client_ids=client_ids, query=query, page_size=2)
+ for i, client_id in enumerate(client_ids):
+ name, args, kwargs = service.search.mock_calls[i]
+ assert kwargs['request'].customer_id == client_id
+ assert kwargs['request'].query == query
+ assert kwargs['request'].page_size == 2
def test_extract_rows(self, mock_hook):
iterators = [[1, 2, 3], [4, 5, 6]]
diff --git a/tests/providers/google/cloud/hooks/test_bigquery.py b/tests/providers/google/cloud/hooks/test_bigquery.py
index 8500694..66d0003 100644
--- a/tests/providers/google/cloud/hooks/test_bigquery.py
+++ b/tests/providers/google/cloud/hooks/test_bigquery.py
@@ -1761,7 +1761,15 @@ class TestBigQueryWithKMS(_BigQueryBaseTestClass):
allow_jagged_rows = False
encoding = "UTF-8"
labels = {'label1': 'test1', 'label2': 'test2'}
- schema_fields = [{'mode': 'REQUIRED', 'name': 'id', 'type': 'STRING', 'description': None}]
+ schema_fields = [
+ {
+ 'mode': 'REQUIRED',
+ 'name': 'id',
+ 'type': 'STRING',
+ 'description': None,
+ 'policyTags': {'names': []},
+ }
+ ]
encryption_configuration = {"kms_key_name": "projects/p/locations/l/keyRings/k/cryptoKeys/c"}
self.hook.create_external_table(