You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2024/08/04 19:51:10 UTC
(airflow) branch main updated: providers/yandex: fix typing (#40997)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 2daa5bd01a providers/yandex: fix typing (#40997)
2daa5bd01a is described below
commit 2daa5bd01a65b6062ec3a38aa71dc462c805ba28
Author: got686-yandex <go...@yandex-team.ru>
AuthorDate: Sun Aug 4 22:51:02 2024 +0300
providers/yandex: fix typing (#40997)
* providers/yandex: fix typing
https://github.com/yandex-cloud/python-sdk/issues/106
* providers/yandex: fix typing
https://github.com/yandex-cloud/python-sdk/issues/106
Fix tests
---
airflow/providers/yandex/hooks/dataproc.py | 22 +++++++++++++++++-
airflow/providers/yandex/hooks/yandex.py | 10 ++++++--
airflow/providers/yandex/hooks/yq.py | 5 ++--
airflow/providers/yandex/operators/dataproc.py | 27 ++++++++++++++--------
airflow/providers/yandex/provider.yaml | 7 +-----
airflow/providers/yandex/secrets/lockbox.py | 2 +-
airflow/providers/yandex/utils/credentials.py | 11 +++++++--
generated/provider_dependencies.json | 2 +-
tests/providers/yandex/hooks/test_dataproc.py | 12 +++++-----
.../system/providers/yandex/example_yandexcloud.py | 19 +++++++++++----
10 files changed, 81 insertions(+), 36 deletions(-)
diff --git a/airflow/providers/yandex/hooks/dataproc.py b/airflow/providers/yandex/hooks/dataproc.py
index 9b1862205e..1b906613ae 100644
--- a/airflow/providers/yandex/hooks/dataproc.py
+++ b/airflow/providers/yandex/hooks/dataproc.py
@@ -16,8 +16,16 @@
# under the License.
from __future__ import annotations
+from typing import TYPE_CHECKING
+
+from deprecated import deprecated
+
+from airflow.exceptions import AirflowProviderDeprecationWarning
from airflow.providers.yandex.hooks.yandex import YandexCloudBaseHook
+if TYPE_CHECKING:
+ from yandexcloud._wrappers.dataproc import Dataproc
+
class DataprocHook(YandexCloudBaseHook):
"""
@@ -29,7 +37,19 @@ class DataprocHook(YandexCloudBaseHook):
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
self.cluster_id = None
- self.client = self.sdk.wrappers.Dataproc(
+ self.dataproc_client: Dataproc = self.sdk.wrappers.Dataproc(
default_folder_id=self.default_folder_id,
default_public_ssh_key=self.default_public_ssh_key,
)
+
+ @property
+ @deprecated(
+ reason="`client` deprecated and will be removed in the future. Use `dataproc_client` instead",
+ category=AirflowProviderDeprecationWarning,
+ )
+ def client(self):
+ return self.dataproc_client
+
+ @client.setter
+ def client(self, value):
+ self.dataproc_client = value
diff --git a/airflow/providers/yandex/hooks/yandex.py b/airflow/providers/yandex/hooks/yandex.py
index 5ad7ce2895..f132b70822 100644
--- a/airflow/providers/yandex/hooks/yandex.py
+++ b/airflow/providers/yandex/hooks/yandex.py
@@ -24,6 +24,7 @@ import yandexcloud
from airflow.exceptions import AirflowProviderDeprecationWarning
from airflow.hooks.base import BaseHook
from airflow.providers.yandex.utils.credentials import (
+ CredentialsType,
get_credentials,
get_service_account_id,
)
@@ -132,13 +133,18 @@ class YandexCloudBaseHook(BaseHook):
self.connection_id = yandex_conn_id or connection_id or default_conn_name
self.connection = self.get_connection(self.connection_id)
self.extras = self.connection.extra_dejson
- self.credentials = get_credentials(
+ self.credentials: CredentialsType = get_credentials(
oauth_token=self._get_field("oauth"),
service_account_json=self._get_field("service_account_json"),
service_account_json_path=self._get_field("service_account_json_path"),
)
sdk_config = self._get_endpoint()
- self.sdk = yandexcloud.SDK(user_agent=provider_user_agent(), **sdk_config, **self.credentials)
+ self.sdk = yandexcloud.SDK(
+ user_agent=provider_user_agent(),
+ token=self.credentials.get("token"),
+ service_account_key=self.credentials.get("service_account_key"),
+ endpoint=sdk_config.get("endpoint"),
+ )
self.default_folder_id = default_folder_id or self._get_field("folder_id")
self.default_public_ssh_key = default_public_ssh_key or self._get_field("public_ssh_key")
self.default_service_account_id = default_service_account_id or get_service_account_id(
diff --git a/airflow/providers/yandex/hooks/yq.py b/airflow/providers/yandex/hooks/yq.py
index 5950c0d6b0..011476b043 100644
--- a/airflow/providers/yandex/hooks/yq.py
+++ b/airflow/providers/yandex/hooks/yq.py
@@ -100,8 +100,7 @@ class YQHook(YandexCloudBaseHook):
return self.client.compose_query_web_link(query_id)
def _get_iam_token(self) -> str:
- iam_token = self.credentials.get("token")
- if iam_token is not None:
- return iam_token
+ if "token" in self.credentials:
+ return self.credentials["token"]
return yc_auth.get_auth_token(service_account_key=self.credentials.get("service_account_key"))
diff --git a/airflow/providers/yandex/operators/dataproc.py b/airflow/providers/yandex/operators/dataproc.py
index 2539314fcd..79aab002e1 100644
--- a/airflow/providers/yandex/operators/dataproc.py
+++ b/airflow/providers/yandex/operators/dataproc.py
@@ -33,7 +33,7 @@ class InitializationAction:
"""Data for initialization action to be run at start of DataProc cluster."""
uri: str # Uri of the executable file
- args: Sequence[str] # Arguments to the initialization action
+ args: Iterable[str] # Arguments to the initialization action
timeout: int # Execution timeout
@@ -143,6 +143,12 @@ class DataprocCreateClusterOperator(BaseOperator):
**kwargs,
) -> None:
super().__init__(**kwargs)
+ if ssh_public_keys is None:
+ ssh_public_keys = []
+
+ if services is None:
+ services = []
+
self.folder_id = folder_id
self.yandex_conn_id = connection_id
self.cluster_name = cluster_name
@@ -186,7 +192,7 @@ class DataprocCreateClusterOperator(BaseOperator):
self.hook = DataprocHook(
yandex_conn_id=self.yandex_conn_id,
)
- operation_result = self.hook.client.create_cluster(
+ operation_result = self.hook.dataproc_client.create_cluster(
folder_id=self.folder_id,
cluster_name=self.cluster_name,
cluster_description=self.cluster_description,
@@ -221,15 +227,16 @@ class DataprocCreateClusterOperator(BaseOperator):
security_group_ids=self.security_group_ids,
log_group_id=self.log_group_id,
labels=self.labels,
- initialization_actions=self.initialization_actions
- and [
+ initialization_actions=[
self.hook.sdk.wrappers.InitializationAction(
uri=init_action.uri,
args=init_action.args,
timeout=init_action.timeout,
)
for init_action in self.initialization_actions
- ],
+ ]
+ if self.initialization_actions
+ else None,
)
cluster_id = operation_result.response.id
@@ -290,7 +297,7 @@ class DataprocDeleteClusterOperator(DataprocBaseOperator):
def execute(self, context: Context) -> None:
hook = self._setup(context)
- hook.client.delete_cluster(self.cluster_id)
+ hook.dataproc_client.delete_cluster(self.cluster_id)
class DataprocCreateHiveJobOperator(DataprocBaseOperator):
@@ -331,7 +338,7 @@ class DataprocCreateHiveJobOperator(DataprocBaseOperator):
def execute(self, context: Context) -> None:
hook = self._setup(context)
- hook.client.create_hive_job(
+ hook.dataproc_client.create_hive_job(
query=self.query,
query_file_uri=self.query_file_uri,
script_variables=self.script_variables,
@@ -387,7 +394,7 @@ class DataprocCreateMapReduceJobOperator(DataprocBaseOperator):
def execute(self, context: Context) -> None:
hook = self._setup(context)
- hook.client.create_mapreduce_job(
+ hook.dataproc_client.create_mapreduce_job(
main_class=self.main_class,
main_jar_file_uri=self.main_jar_file_uri,
jar_file_uris=self.jar_file_uris,
@@ -455,7 +462,7 @@ class DataprocCreateSparkJobOperator(DataprocBaseOperator):
def execute(self, context: Context) -> None:
hook = self._setup(context)
- hook.client.create_spark_job(
+ hook.dataproc_client.create_spark_job(
main_class=self.main_class,
main_jar_file_uri=self.main_jar_file_uri,
jar_file_uris=self.jar_file_uris,
@@ -526,7 +533,7 @@ class DataprocCreatePysparkJobOperator(DataprocBaseOperator):
def execute(self, context: Context) -> None:
hook = self._setup(context)
- hook.client.create_pyspark_job(
+ hook.dataproc_client.create_pyspark_job(
main_python_file_uri=self.main_python_file_uri,
python_file_uris=self.python_file_uris,
jar_file_uris=self.jar_file_uris,
diff --git a/airflow/providers/yandex/provider.yaml b/airflow/providers/yandex/provider.yaml
index 5ec7bd3939..d6dd9aa7aa 100644
--- a/airflow/providers/yandex/provider.yaml
+++ b/airflow/providers/yandex/provider.yaml
@@ -52,12 +52,7 @@ versions:
dependencies:
- apache-airflow>=2.7.0
- # The 0.289 and 0.290 versions have broken dataproc support
- # See https://github.com/yandex-cloud/python-sdk/issues/103
- # the 0.291.0 version of yandex provider introduced mypy typing
- # that conflicts with the way yandex provider uses it and should be fixed
- # See https://github.com/yandex-cloud/python-sdk/issues/106
- - yandexcloud>=0.278.0,!=0.289.0,!=0.290.0,<0.292.0
+ - yandexcloud>=0.308.0
- yandex-query-client>=0.1.4
integrations:
diff --git a/airflow/providers/yandex/secrets/lockbox.py b/airflow/providers/yandex/secrets/lockbox.py
index 68879b446e..d65131ab2c 100644
--- a/airflow/providers/yandex/secrets/lockbox.py
+++ b/airflow/providers/yandex/secrets/lockbox.py
@@ -242,7 +242,7 @@ class LockboxSecretBackend(BaseSecretsBackend, LoggingMixin):
return f"{prefix}{self.sep}{key}"
def _get_secret_value(self, prefix: str, key: str) -> str | None:
- secret: secret_pb.Secret = None
+ secret: secret_pb.Secret | None = None
for s in self._get_secrets():
if s.name == self._build_secret_name(prefix=prefix, key=key):
secret = s
diff --git a/airflow/providers/yandex/utils/credentials.py b/airflow/providers/yandex/utils/credentials.py
index f54e8bdfbe..015e30ceaa 100644
--- a/airflow/providers/yandex/utils/credentials.py
+++ b/airflow/providers/yandex/utils/credentials.py
@@ -18,16 +18,23 @@ from __future__ import annotations
import json
import logging
-from typing import Any
+from typing import TypedDict
log = logging.getLogger(__name__)
+class CredentialsType(TypedDict, total=False):
+ """Credentials dict description."""
+
+ token: str
+ service_account_key: dict[str, str]
+
+
def get_credentials(
oauth_token: str | None = None,
service_account_json: dict | str | None = None,
service_account_json_path: str | None = None,
-) -> dict[str, Any]:
+) -> CredentialsType:
"""
Return credentials JSON for Yandex Cloud SDK based on credentials.
diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json
index 35f7ea034a..ed1e4ab7b8 100644
--- a/generated/provider_dependencies.json
+++ b/generated/provider_dependencies.json
@@ -1348,7 +1348,7 @@
"deps": [
"apache-airflow>=2.7.0",
"yandex-query-client>=0.1.4",
- "yandexcloud>=0.278.0,!=0.289.0,!=0.290.0,<0.292.0"
+ "yandexcloud>=0.308.0"
],
"devel-deps": [],
"plugins": [],
diff --git a/tests/providers/yandex/hooks/test_dataproc.py b/tests/providers/yandex/hooks/test_dataproc.py
index 0665b17b47..69c0630c99 100644
--- a/tests/providers/yandex/hooks/test_dataproc.py
+++ b/tests/providers/yandex/hooks/test_dataproc.py
@@ -70,7 +70,7 @@ class TestYandexCloudDataprocHook:
def test_create_dataproc_cluster_mocked(self, mock_create_operation):
self._init_hook()
- self.hook.client.create_cluster(
+ self.hook.dataproc_client.create_cluster(
cluster_name=CLUSTER_NAME,
ssh_public_keys=SSH_PUBLIC_KEYS,
folder_id=FOLDER_ID,
@@ -85,14 +85,14 @@ class TestYandexCloudDataprocHook:
@mock.patch("yandexcloud.SDK.create_operation_and_get_result")
def test_delete_dataproc_cluster_mocked(self, mock_create_operation):
self._init_hook()
- self.hook.client.delete_cluster("my_cluster_id")
+ self.hook.dataproc_client.delete_cluster("my_cluster_id")
assert mock_create_operation.called
@mock.patch("yandexcloud.SDK.create_operation_and_get_result")
def test_create_hive_job_hook(self, mock_create_operation):
self._init_hook()
- self.hook.client.create_hive_job(
+ self.hook.dataproc_client.create_hive_job(
cluster_id="my_cluster_id",
continue_on_failure=False,
name="Hive job",
@@ -106,7 +106,7 @@ class TestYandexCloudDataprocHook:
def test_create_mapreduce_job_hook(self, mock_create_operation):
self._init_hook()
- self.hook.client.create_mapreduce_job(
+ self.hook.dataproc_client.create_mapreduce_job(
archive_uris=None,
args=[
"-mapper",
@@ -141,7 +141,7 @@ class TestYandexCloudDataprocHook:
def test_create_spark_job_hook(self, mock_create_operation):
self._init_hook()
- self.hook.client.create_spark_job(
+ self.hook.dataproc_client.create_spark_job(
archive_uris=["s3a://some-in-bucket/jobs/sources/data/country-codes.csv.zip"],
args=[
"s3a://some-in-bucket/jobs/sources/data/cities500.txt.bz2",
@@ -166,7 +166,7 @@ class TestYandexCloudDataprocHook:
def test_create_pyspark_job_hook(self, mock_create_operation):
self._init_hook()
- self.hook.client.create_pyspark_job(
+ self.hook.dataproc_client.create_pyspark_job(
archive_uris=["s3a://some-in-bucket/jobs/sources/data/country-codes.csv.zip"],
args=[
"s3a://some-in-bucket/jobs/sources/data/cities500.txt.bz2",
diff --git a/tests/system/providers/yandex/example_yandexcloud.py b/tests/system/providers/yandex/example_yandexcloud.py
index 2751458ae9..ddebc46a3b 100644
--- a/tests/system/providers/yandex/example_yandexcloud.py
+++ b/tests/system/providers/yandex/example_yandexcloud.py
@@ -27,6 +27,7 @@ import yandex.cloud.dataproc.v1.job_service_pb2 as job_service_pb
import yandex.cloud.dataproc.v1.job_service_pb2_grpc as job_service_grpc_pb
import yandex.cloud.dataproc.v1.subcluster_pb2 as subcluster_pb
from google.protobuf.json_format import MessageToDict
+from yandexcloud.operations import OperationError
from airflow import DAG
from airflow.decorators import task
@@ -61,7 +62,7 @@ def create_cluster_request(
bucket=YC_S3_BUCKET_NAME,
config_spec=cluster_service_pb.CreateClusterConfigSpec(
hadoop=cluster_pb.HadoopConfig(
- services=("SPARK", "YARN"),
+ services=(cluster_pb.HadoopConfig.Service.SPARK, cluster_pb.HadoopConfig.Service.YARN),
ssh_public_keys=[ssh_public_key],
),
subclusters_spec=[
@@ -98,13 +99,13 @@ def create_cluster(
*,
dag: DAG | None = None,
ts_nodash: str | None = None,
-) -> str:
+) -> str | None:
hook = YandexCloudBaseHook(yandex_conn_id=yandex_conn_id)
folder_id = folder_id or hook.default_folder_id
if subnet_id is None:
network_id = network_id or hook.sdk.helpers.find_network_id(folder_id)
subnet_id = hook.sdk.helpers.find_subnet_id(folder_id=folder_id, zone_id=zone, network_id=network_id)
- service_account_id = service_account_id or hook.sdk.helpers.find_service_account_id()
+ service_account_id = service_account_id or hook.sdk.helpers.find_service_account_id(folder_id=folder_id)
ssh_public_key = ssh_public_key or hook.default_public_ssh_key
dag_id = dag and dag.dag_id or "dag"
@@ -126,6 +127,12 @@ def create_cluster(
operation_result = hook.sdk.wait_operation_and_get_result(
operation, response_type=cluster_pb.Cluster, meta_type=cluster_service_pb.CreateClusterMetadata
)
+ if isinstance(operation_result, OperationError):
+ raise ValueError("Cluster creation error")
+
+ if operation_result.response is None:
+ return None
+
return operation_result.response.id
@@ -149,7 +156,11 @@ def run_spark_job(
operation_result = hook.sdk.wait_operation_and_get_result(
operation, response_type=job_pb.Job, meta_type=job_service_pb.CreateJobMetadata
)
- return MessageToDict(operation_result.response)
+
+ if isinstance(operation_result, OperationError):
+ raise ValueError("Run spark task error")
+
+ return MessageToDict(operation_result.response) if operation_result.response is not None else None
@task(trigger_rule="all_done")