You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2026/05/10 03:26:48 UTC
(airflow) branch main updated: Remove MLEngine from google provider (#66526)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new e6bd1e23bca Remove MLEngine from google provider (#66526)
e6bd1e23bca is described below
commit e6bd1e23bca438f15a8ee3eef73264c8dd5c85b1
Author: Nitochkin <62...@users.noreply.github.com>
AuthorDate: Sun May 10 05:26:39 2026 +0200
Remove MLEngine from google provider (#66526)
Co-authored-by: Anton Nitochkin <ni...@google.com>
---
.../tests/unit/always/test_project_structure.py | 2 -
providers/google/docs/changelog.rst | 17 +
.../google/docs/integration-logos/AI-Platform.png | Bin 6284 -> 0 bytes
providers/google/docs/operators/cloud/mlengine.rst | 86 --
providers/google/provider.yaml | 17 -
.../providers/google/cloud/hooks/mlengine.py | 605 -----------
.../providers/google/cloud/links/mlengine.py | 71 --
.../providers/google/cloud/triggers/mlengine.py | 125 ---
.../airflow/providers/google/get_provider_info.py | 20 -
.../tests/unit/google/cloud/hooks/test_mlengine.py | 1128 --------------------
.../unit/google/cloud/triggers/test_mlengine.py | 142 ---
11 files changed, 17 insertions(+), 2196 deletions(-)
diff --git a/airflow-core/tests/unit/always/test_project_structure.py b/airflow-core/tests/unit/always/test_project_structure.py
index 13ef1ee8320..750925072d2 100644
--- a/airflow-core/tests/unit/always/test_project_structure.py
+++ b/airflow-core/tests/unit/always/test_project_structure.py
@@ -158,7 +158,6 @@ class TestProjectStructure:
"providers/google/tests/unit/google/cloud/links/test_dataproc.py",
"providers/google/tests/unit/google/cloud/links/test_datastore.py",
"providers/google/tests/unit/google/cloud/links/test_kubernetes_engine.py",
- "providers/google/tests/unit/google/cloud/links/test_mlengine.py",
"providers/google/tests/unit/google/cloud/links/test_pubsub.py",
"providers/google/tests/unit/google/cloud/links/test_spanner.py",
"providers/google/tests/unit/google/cloud/links/test_stackdriver.py",
@@ -420,7 +419,6 @@ class TestGoogleProviderProjectStructure(ExampleCoverageTest, AssetsCoverageTest
"airflow.providers.google.cloud.operators.bigquery.BigQueryCreateExternalTableOperator",
"airflow.providers.google.cloud.operators.datapipeline.CreateDataPipelineOperator",
"airflow.providers.google.cloud.operators.datapipeline.RunDataPipelineOperator",
- "airflow.providers.google.cloud.operators.mlengine.MLEngineCreateModelOperator",
"airflow.providers.google.cloud.operators.vertex_ai.generative_model.TextGenerationModelPredictOperator",
"airflow.providers.google.marketing_platform.operators.GoogleDisplayVideo360CreateQueryOperator",
"airflow.providers.google.marketing_platform.operators.GoogleDisplayVideo360DeleteReportOperator",
diff --git a/providers/google/docs/changelog.rst b/providers/google/docs/changelog.rst
index 6d9a711b138..b8e107bda9c 100644
--- a/providers/google/docs/changelog.rst
+++ b/providers/google/docs/changelog.rst
@@ -27,6 +27,23 @@
Changelog
---------
+.. warning::
+ Deprecated classes, parameters and features have been removed from the Google provider package.
+ The following breaking changes were introduced:
+
+* Hooks
+ * Remove ``MLEngineHook`` use ``airflow.providers.google.cloud.hooks.vertex_ai.ModelServiceHook`` instead
+ * Remove ``MLEngineAsyncHook``
+
+* Triggers
+ * Remove ``MLEngineStartTrainingJobTrigger`` use appropriate trigger from ``airflow.providers.google.cloud.triggers.vertex_ai``
+
+* Links
+ * Remove ``MLEngineModelLink`` use ``airflow.providers.google.cloud.links.vertex_ai.VertexAIModelLink`` instead
+ * Remove ``MLEngineModelsListLink`` use ``airflow.providers.google.cloud.links.vertex_ai.VertexAIModelListLink`` instead
+ * Remove ``MLEngineJobDetailsLink`` use appropriate link from ``airflow.providers.google.cloud.links.vertex_ai`` instead
+ * Remove ``MLEngineModelVersionDetailsLink`` use ``airflow.providers.google.cloud.links.vertex_ai.VertexAIModelLink`` instead
+ * Remove ``MLEngineJobSListLink`` use appropriate link from ``airflow.providers.google.cloud.links.vertex_ai`` instead
21.3.0
......
diff --git a/providers/google/docs/integration-logos/AI-Platform.png b/providers/google/docs/integration-logos/AI-Platform.png
deleted file mode 100644
index f85ff7f8e3d..00000000000
Binary files a/providers/google/docs/integration-logos/AI-Platform.png and /dev/null differ
diff --git a/providers/google/docs/operators/cloud/mlengine.rst b/providers/google/docs/operators/cloud/mlengine.rst
deleted file mode 100644
index 407954011ef..00000000000
--- a/providers/google/docs/operators/cloud/mlengine.rst
+++ /dev/null
@@ -1,86 +0,0 @@
- .. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- .. http://www.apache.org/licenses/LICENSE-2.0
-
- .. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
-
-
-Google Cloud AI Platform Operators
-==================================
-
-`Google Cloud AI Platform <https://cloud.google.com/ai-platform/>`__ (formerly known
-as ML Engine) can be used to train machine learning models at scale, host trained models
-in the cloud, and use models to make predictions for new data. AI Platform is a collection
-of tools for training, evaluating, and tuning machine learning models. AI Platform can also
-be used to deploy a trained model, make predictions, and manage various model versions.
-
-The legacy versions of AI Platform Training, AI Platform Prediction, AI Platform Pipelines,
-and AI Platform Data Labeling Service are deprecated and will no longer be available on
-Google Cloud after their shutdown date. All the functionality of legacy AI Platform and new
-features are available on the Vertex AI platform.
-
-Prerequisite tasks
-^^^^^^^^^^^^^^^^^^
-
-.. include:: /operators/_partials/prerequisite_tasks.rst
-
-Launching a Job
-^^^^^^^^^^^^^^^
-This function is deprecated. All the functionality of legacy MLEngine and new features are available on
-the Vertex AI platform.
-
-.. _howto/operator:MLEngineCreateModelOperator:
-
-Creating a model
-^^^^^^^^^^^^^^^^
-This function is deprecated. All the functionality of legacy MLEngine and new features are available on
-the Vertex AI platform.
-
-Getting a model
-^^^^^^^^^^^^^^^
-This function is deprecated. All the functionality of legacy MLEngine and new features are available on
-the Vertex AI platform.
-
-Creating model versions
-^^^^^^^^^^^^^^^^^^^^^^^
-
-This function is deprecated. All the functionality of legacy MLEngine and new features are available on
-the Vertex AI platform.
-
-Managing model versions
-^^^^^^^^^^^^^^^^^^^^^^^
-
-This function is deprecated. All the functionality of legacy MLEngine and new features are available on
-the Vertex AI platform.
-
-
-Making predictions
-^^^^^^^^^^^^^^^^^^
-
-This function is deprecated. All the functionality of legacy MLEngine and new features are available on
-the Vertex AI platform.
-
-
-Cleaning up
-^^^^^^^^^^^
-
-This function is deprecated. All the functionality of legacy MLEngine and new features are available on
-the Vertex AI platform.
-
-Evaluating a model
-^^^^^^^^^^^^^^^^^^
-
-This function is deprecated. All the functionality of legacy MLEngine and new features are available on
-the Vertex AI platform.
diff --git a/providers/google/provider.yaml b/providers/google/provider.yaml
index bd6f5fd73c4..aff958b5429 100644
--- a/providers/google/provider.yaml
+++ b/providers/google/provider.yaml
@@ -418,12 +418,6 @@ integrations:
- /docs/apache-airflow-providers-google/operators/cloud/kubernetes_engine.rst
logo: /docs/integration-logos/Kubernetes-Engine.png
tags: [gcp]
- - integration-name: Google Machine Learning Engine
- external-doc-url: https://cloud.google.com/vertex-ai/
- how-to-guide:
- - /docs/apache-airflow-providers-google/operators/cloud/mlengine.rst
- logo: /docs/integration-logos/AI-Platform.png
- tags: [gcp]
- integration-name: Google Cloud Natural Language
external-doc-url: https://cloud.google.com/natural-language/
how-to-guide:
@@ -817,9 +811,6 @@ hooks:
- integration-name: Google Kubernetes Engine
python-modules:
- airflow.providers.google.cloud.hooks.kubernetes_engine
- - integration-name: Google Machine Learning Engine
- python-modules:
- - airflow.providers.google.cloud.hooks.mlengine
- integration-name: Google Cloud Natural Language
python-modules:
- airflow.providers.google.cloud.hooks.natural_language
@@ -978,9 +969,6 @@ triggers:
- integration-name: Google Kubernetes Engine
python-modules:
- airflow.providers.google.cloud.triggers.kubernetes_engine
- - integration-name: Google Machine Learning Engine
- python-modules:
- - airflow.providers.google.cloud.triggers.mlengine
- integration-name: Google Cloud Pub/Sub
python-modules:
- airflow.providers.google.cloud.triggers.pubsub
@@ -1451,11 +1439,6 @@ extra-links:
- airflow.providers.google.cloud.links.data_loss_prevention.CloudDLPInfoTypesListLink
- airflow.providers.google.cloud.links.data_loss_prevention.CloudDLPInfoTypeDetailsLink
- airflow.providers.google.cloud.links.data_loss_prevention.CloudDLPPossibleInfoTypesListLink
- - airflow.providers.google.cloud.links.mlengine.MLEngineModelLink
- - airflow.providers.google.cloud.links.mlengine.MLEngineModelsListLink
- - airflow.providers.google.cloud.links.mlengine.MLEngineJobDetailsLink
- - airflow.providers.google.cloud.links.mlengine.MLEngineJobSListLink
- - airflow.providers.google.cloud.links.mlengine.MLEngineModelVersionDetailsLink
- airflow.providers.google.common.links.storage.StorageLink
- airflow.providers.google.common.links.storage.FileDetailsLink
- airflow.providers.google.marketing_platform.links.analytics_admin.GoogleAnalyticsPropertyLink
diff --git a/providers/google/src/airflow/providers/google/cloud/hooks/mlengine.py b/providers/google/src/airflow/providers/google/cloud/hooks/mlengine.py
deleted file mode 100644
index 70a420d1351..00000000000
--- a/providers/google/src/airflow/providers/google/cloud/hooks/mlengine.py
+++ /dev/null
@@ -1,605 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""This module contains a Google ML Engine Hook."""
-
-from __future__ import annotations
-
-import contextlib
-import logging
-import random
-import time
-from collections.abc import Callable
-from typing import TYPE_CHECKING
-
-from aiohttp import ClientSession
-from gcloud.aio.auth import AioSession, Token
-from googleapiclient.discovery import Resource, build
-from googleapiclient.errors import HttpError
-
-from airflow.providers.common.compat.sdk import AirflowException
-from airflow.providers.google.common.hooks.base_google import (
- PROVIDE_PROJECT_ID,
- GoogleBaseAsyncHook,
- GoogleBaseHook,
-)
-from airflow.version import version as airflow_version
-
-if TYPE_CHECKING:
- from httplib2 import Response
- from requests import Session
-
-log = logging.getLogger(__name__)
-
-_AIRFLOW_VERSION = "v" + airflow_version.replace(".", "-").replace("+", "-")
-
-
-def _poll_with_exponential_delay(
- request, execute_num_retries, max_n, is_done_func, is_error_func
-) -> Response:
- """
- Execute request with exponential delay.
-
- This method is intended to handle and retry in case of api-specific errors,
- such as 429 "Too Many Requests", unlike the `request.execute` which handles
- lower level errors like `ConnectionError`/`socket.timeout`/`ssl.SSLError`.
-
- :param request: request to be executed.
- :param execute_num_retries: num_retries for `request.execute` method.
- :param max_n: number of times to retry request in this method.
- :param is_done_func: callable to determine if operation is done.
- :param is_error_func: callable to determine if operation is failed.
- :return: response
- """
- for i in range(0, max_n):
- try:
- response = request.execute(num_retries=execute_num_retries)
- if is_error_func(response):
- raise ValueError(f"The response contained an error: {response}")
- if is_done_func(response):
- log.info("Operation is done: %s", response)
- return response
-
- time.sleep((2**i) + random.random())
- except HttpError as e:
- if e.resp.status != 429:
- log.info("Something went wrong. Not retrying: %s", format(e))
- raise
- time.sleep((2**i) + random.random())
-
- raise ValueError(f"Connection could not be established after {max_n} retries.")
-
-
-class MLEngineHook(GoogleBaseHook):
- """
- Hook for Google ML Engine APIs.
-
- All the methods in the hook where project_id is used must be called with
- keyword arguments rather than positional.
- """
-
- def get_conn(self) -> Resource:
- """
- Retrieve the connection to MLEngine.
-
- :return: Google MLEngine services object.
- """
- authed_http = self._authorize()
- return build("ml", "v1", http=authed_http, cache_discovery=False)
-
- @GoogleBaseHook.fallback_to_default_project_id
- def create_job(self, job: dict, project_id: str, use_existing_job_fn: Callable | None = None) -> dict:
- """
- Launch a MLEngine job and wait for it to reach a terminal state.
-
- :param project_id: The Google Cloud project id within which MLEngine
- job will be launched. If set to None or missing, the default project_id from the Google Cloud
- connection is used.
- :param job: MLEngine Job object that should be provided to the MLEngine
- API, such as: ::
-
- {
- 'jobId': 'my_job_id',
- 'trainingInput': {
- 'scaleTier': 'STANDARD_1',
- ...
- }
- }
-
- :param use_existing_job_fn: In case that a MLEngine job with the same
- job_id already exist, this method (if provided) will decide whether
- we should use this existing job, continue waiting for it to finish
- and returning the job object. It should accepts a MLEngine job
- object, and returns a boolean value indicating whether it is OK to
- reuse the existing job. If 'use_existing_job_fn' is not provided,
- we by default reuse the existing MLEngine job.
- :return: The MLEngine job object if the job successfully reach a
- terminal state (which might be FAILED or CANCELLED state).
- """
- hook = self.get_conn()
-
- self._append_label(job)
- self.log.info("Creating job.")
-
- request = hook.projects().jobs().create(parent=f"projects/{project_id}", body=job)
- job_id = job["jobId"]
-
- try:
- request.execute(num_retries=self.num_retries)
- except HttpError as e:
- # 409 means there is an existing job with the same job ID.
- if e.resp.status == 409:
- if use_existing_job_fn is not None:
- existing_job = self.get_job(project_id, job_id)
- if not use_existing_job_fn(existing_job):
- self.log.error(
- "Job with job_id %s already exist, but it does not match our expectation: %s",
- job_id,
- existing_job,
- )
- raise
- self.log.info("Job with job_id %s already exist. Will waiting for it to finish", job_id)
- else:
- self.log.error("Failed to create MLEngine job: %s", e)
- raise
-
- return self._wait_for_job_done(project_id, job_id)
-
- @GoogleBaseHook.fallback_to_default_project_id
- def create_job_without_waiting_result(
- self,
- body: dict,
- project_id: str,
- ):
- """
- Launch a MLEngine job and wait for it to reach a terminal state.
-
- :param project_id: The Google Cloud project id within which MLEngine
- job will be launched. If set to None or missing, the default project_id from the Google Cloud
- connection is used.
- :param body: MLEngine Job object that should be provided to the MLEngine
- API, such as: ::
-
- {
- 'jobId': 'my_job_id',
- 'trainingInput': {
- 'scaleTier': 'STANDARD_1',
- ...
- }
- }
- :return: The MLEngine job_id of the object if the job successfully reach a
- terminal state (which might be FAILED or CANCELLED state).
- """
- hook = self.get_conn()
-
- self._append_label(body)
- request = hook.projects().jobs().create(parent=f"projects/{project_id}", body=body)
- job_id = body["jobId"]
- request.execute(num_retries=self.num_retries)
- return job_id
-
- @GoogleBaseHook.fallback_to_default_project_id
- def cancel_job(
- self,
- job_id: str,
- project_id: str,
- ) -> dict:
- """
- Cancel a MLEngine job.
-
- :param project_id: The Google Cloud project id within which MLEngine
- job will be cancelled. If set to None or missing, the default project_id from the Google Cloud
- connection is used.
- :param job_id: A unique id for the want-to-be cancelled Google MLEngine training job.
-
- :return: Empty dict if cancelled successfully
- :raises: googleapiclient.errors.HttpError
- """
- hook = self.get_conn()
-
- request = hook.projects().jobs().cancel(name=f"projects/{project_id}/jobs/{job_id}")
-
- try:
- return request.execute(num_retries=self.num_retries)
- except HttpError as e:
- if e.resp.status == 404:
- self.log.error("Job with job_id %s does not exist. ", job_id)
- raise
- if e.resp.status == 400:
- self.log.info("Job with job_id %s is already complete, cancellation aborted.", job_id)
- return {}
- self.log.error("Failed to cancel MLEngine job: %s", e)
- raise
-
- def get_job(self, project_id: str, job_id: str) -> dict:
- """
- Get a MLEngine job based on the job id.
-
- :param project_id: The project in which the Job is located. If set to None or missing, the default
- project_id from the Google Cloud connection is used. (templated)
- :param job_id: A unique id for the Google MLEngine job. (templated)
- :return: MLEngine job object if succeed.
- :raises: googleapiclient.errors.HttpError
- """
- hook = self.get_conn()
- job_name = f"projects/{project_id}/jobs/{job_id}"
- request = hook.projects().jobs().get(name=job_name)
- while True:
- try:
- return request.execute(num_retries=self.num_retries)
- except HttpError as e:
- if e.resp.status == 429:
- # polling after 30 seconds when quota failure occurs
- time.sleep(30)
- else:
- self.log.error("Failed to get MLEngine job: %s", e)
- raise
-
- def _wait_for_job_done(self, project_id: str, job_id: str, interval: int = 30):
- """
- Wait for the Job to reach a terminal state.
-
- This method will periodically check the job state until the job reach
- a terminal state.
-
- :param project_id: The project in which the Job is located. If set to None or missing, the default
- project_id from the Google Cloud connection is used. (templated)
- :param job_id: A unique id for the Google MLEngine job. (templated)
- :param interval: Time expressed in seconds after which the job status is checked again. (templated)
- :raises: googleapiclient.errors.HttpError
- """
- self.log.info("Waiting for job. job_id=%s", job_id)
-
- if interval <= 0:
- raise ValueError("Interval must be > 0")
- while True:
- job = self.get_job(project_id, job_id)
- if job["state"] in ["SUCCEEDED", "FAILED", "CANCELLED"]:
- return job
- time.sleep(interval)
-
- @GoogleBaseHook.fallback_to_default_project_id
- def create_version(
- self,
- model_name: str,
- version_spec: dict,
- project_id: str,
- ) -> dict:
- """
- Create the Version on Google Cloud ML Engine.
-
- :param version_spec: A dictionary containing the information about the version. (templated)
- :param model_name: The name of the Google Cloud ML Engine model that the version belongs to.
- (templated)
- :param project_id: The Google Cloud project name to which MLEngine model belongs.
- If set to None or missing, the default project_id from the Google Cloud connection is used.
- (templated)
- :return: If the version was created successfully, returns the operation.
- Otherwise raises an error .
- """
- hook = self.get_conn()
- parent_name = f"projects/{project_id}/models/{model_name}"
-
- self._append_label(version_spec)
-
- create_request = hook.projects().models().versions().create(parent=parent_name, body=version_spec)
- response = create_request.execute(num_retries=self.num_retries)
- get_request = hook.projects().operations().get(name=response["name"])
-
- return _poll_with_exponential_delay(
- request=get_request,
- execute_num_retries=self.num_retries,
- max_n=9,
- is_done_func=lambda resp: resp.get("done", False),
- is_error_func=lambda resp: resp.get("error", None) is not None,
- )
-
- @GoogleBaseHook.fallback_to_default_project_id
- def set_default_version(
- self,
- model_name: str,
- version_name: str,
- project_id: str,
- ) -> dict:
- """
- Set a version to be the default. Blocks until finished.
-
- :param model_name: The name of the Google Cloud ML Engine model that the version belongs to.
- (templated)
- :param version_name: A name to use for the version being operated upon. (templated)
- :param project_id: The Google Cloud project name to which MLEngine model belongs. If set to None
- or missing, the default project_id from the Google Cloud connection is used. (templated)
- :return: If successful, return an instance of Version.
- Otherwise raises an error.
- :raises: googleapiclient.errors.HttpError
- """
- hook = self.get_conn()
- full_version_name = f"projects/{project_id}/models/{model_name}/versions/{version_name}"
-
- request = hook.projects().models().versions().setDefault(name=full_version_name, body={})
-
- try:
- response = request.execute(num_retries=self.num_retries)
- self.log.info("Successfully set version: %s to default", response)
- return response
- except HttpError as e:
- self.log.error("Something went wrong: %s", e)
- raise
-
- @GoogleBaseHook.fallback_to_default_project_id
- def list_versions(
- self,
- model_name: str,
- project_id: str,
- ) -> list[dict]:
- """
- List all available versions of a model. Blocks until finished.
-
- :param model_name: The name of the Google Cloud ML Engine model that the version
- belongs to. (templated)
- :param project_id: The Google Cloud project name to which MLEngine model belongs. If set to None or
- missing, the default project_id from the Google Cloud connection is used. (templated)
- :return: return an list of instance of Version.
- :raises: googleapiclient.errors.HttpError
- """
- hook = self.get_conn()
- result: list[dict] = []
- full_parent_name = f"projects/{project_id}/models/{model_name}"
-
- request = hook.projects().models().versions().list(parent=full_parent_name, pageSize=100)
-
- while request is not None:
- response = request.execute(num_retries=self.num_retries)
- result.extend(response.get("versions", []))
-
- request = (
- hook.projects()
- .models()
- .versions()
- .list_next(previous_request=request, previous_response=response)
- )
- time.sleep(5)
- return result
-
- @GoogleBaseHook.fallback_to_default_project_id
- def delete_version(
- self,
- model_name: str,
- version_name: str,
- project_id: str,
- ) -> dict:
- """
- Delete the given version of a model. Blocks until finished.
-
- :param model_name: The name of the Google Cloud ML Engine model that the version
- belongs to. (templated)
- :param project_id: The Google Cloud project name to which MLEngine
- model belongs.
- :param version_name: A name to use for the version being operated upon. (templated)
- :return: If the version was deleted successfully, returns the operation.
- Otherwise raises an error.
- """
- hook = self.get_conn()
- full_name = f"projects/{project_id}/models/{model_name}/versions/{version_name}"
- delete_request = hook.projects().models().versions().delete(name=full_name)
- response = delete_request.execute(num_retries=self.num_retries)
- get_request = hook.projects().operations().get(name=response["name"])
-
- return _poll_with_exponential_delay(
- request=get_request,
- execute_num_retries=self.num_retries,
- max_n=9,
- is_done_func=lambda resp: resp.get("done", False),
- is_error_func=lambda resp: resp.get("error", None) is not None,
- )
-
- @GoogleBaseHook.fallback_to_default_project_id
- def create_model(
- self,
- model: dict,
- project_id: str,
- ) -> dict:
- """
- Create a Model. Blocks until finished.
-
- :param model: A dictionary containing the information about the model.
- :param project_id: The Google Cloud project name to which MLEngine model belongs. If set to None or
- missing, the default project_id from the Google Cloud connection is used. (templated)
- :return: If the version was created successfully, returns the instance of Model.
- Otherwise raises an error.
- :raises: googleapiclient.errors.HttpError
- """
- hook = self.get_conn()
- if "name" not in model or not model["name"]:
- raise ValueError("Model name must be provided and could not be an empty string")
- project = f"projects/{project_id}"
-
- self._append_label(model)
- try:
- request = hook.projects().models().create(parent=project, body=model)
- response = request.execute(num_retries=self.num_retries)
- except HttpError as e:
- if e.resp.status != 409:
- raise e
- str(e) # Fills in the error_details field
- if not e.error_details or len(e.error_details) != 1:
- raise e
-
- error_detail = e.error_details[0]
- if error_detail["@type"] != "type.googleapis.com/google.rpc.BadRequest":
- raise e
-
- if "fieldViolations" not in error_detail or len(error_detail["fieldViolations"]) != 1:
- raise e
-
- field_violation = error_detail["fieldViolations"][0]
- if (
- field_violation["field"] != "model.name"
- or field_violation["description"] != "A model with the same name already exists."
- ):
- raise e
- response = self.get_model(model_name=model["name"], project_id=project_id)
-
- return response
-
- @GoogleBaseHook.fallback_to_default_project_id
- def get_model(
- self,
- model_name: str,
- project_id: str,
- ) -> dict | None:
- """
- Get a Model. Blocks until finished.
-
- :param model_name: The name of the model.
- :param project_id: The Google Cloud project name to which MLEngine model belongs. If set to None
- or missing, the default project_id from the Google Cloud connection is used. (templated)
- :return: If the model exists, returns the instance of Model.
- Otherwise return None.
- :raises: googleapiclient.errors.HttpError
- """
- hook = self.get_conn()
- if not model_name:
- raise ValueError("Model name must be provided and it could not be an empty string")
- full_model_name = f"projects/{project_id}/models/{model_name}"
- request = hook.projects().models().get(name=full_model_name)
- try:
- return request.execute(num_retries=self.num_retries)
- except HttpError as e:
- if e.resp.status == 404:
- self.log.error("Model was not found: %s", e)
- return None
- raise
-
- @GoogleBaseHook.fallback_to_default_project_id
- def delete_model(
- self,
- model_name: str,
- project_id: str,
- delete_contents: bool = False,
- ) -> None:
- """
- Delete a Model. Blocks until finished.
-
- :param model_name: The name of the model.
- :param delete_contents: Whether to force the deletion even if the models is not empty.
- Will delete all version (if any) in the dataset if set to True.
- The default value is False.
- :param project_id: The Google Cloud project name to which MLEngine model belongs. If set to None
- or missing, the default project_id from the Google Cloud connection is used. (templated)
- :raises: googleapiclient.errors.HttpError
- """
- hook = self.get_conn()
-
- if not model_name:
- raise ValueError("Model name must be provided and it could not be an empty string")
- model_path = f"projects/{project_id}/models/{model_name}"
- if delete_contents:
- self._delete_all_versions(model_name, project_id)
- request = hook.projects().models().delete(name=model_path)
- try:
- request.execute(num_retries=self.num_retries)
- except HttpError as e:
- if e.resp.status == 404:
- self.log.error("Model was not found: %s", e)
- return
- raise
-
- def _delete_all_versions(self, model_name: str, project_id: str):
- versions = self.list_versions(project_id=project_id, model_name=model_name)
- # The default version can only be deleted when it is the last one in the model
- non_default_versions = (version for version in versions if not version.get("isDefault", False))
- for version in non_default_versions:
- _, _, version_name = version["name"].rpartition("/")
- self.delete_version(project_id=project_id, model_name=model_name, version_name=version_name)
- default_versions = (version for version in versions if version.get("isDefault", False))
- for version in default_versions:
- _, _, version_name = version["name"].rpartition("/")
- self.delete_version(project_id=project_id, model_name=model_name, version_name=version_name)
-
- def _append_label(self, model: dict) -> None:
- model["labels"] = model.get("labels", {})
- model["labels"]["airflow-version"] = _AIRFLOW_VERSION
-
-
-class MLEngineAsyncHook(GoogleBaseAsyncHook):
- """Class to get asynchronous hook for MLEngine."""
-
- sync_hook_class = MLEngineHook
- scopes = ["https://www.googleapis.com/auth/cloud-platform"]
-
- def _check_fileds(
- self,
- job_id: str,
- project_id: str = PROVIDE_PROJECT_ID,
- ):
- if not project_id:
- raise AirflowException("Google Cloud project id is required.")
- if not job_id:
- raise AirflowException("An unique job id is required for Google MLEngine training job.")
-
- async def _get_link(self, url: str, session: Session):
- async with Token(scopes=self.scopes) as token:
- session_aio = AioSession(session)
- headers = {
- "Authorization": f"Bearer {await token.get()}",
- }
- with contextlib.suppress(AirflowException):
- # suppress AirflowException because we don't want to raise exception
- job = await session_aio.get(url=url, headers=headers)
-
- return job
-
- async def get_job(self, job_id: str, session: Session, project_id: str = PROVIDE_PROJECT_ID):
- """Get the specified job resource by job ID and project ID."""
- self._check_fileds(project_id=project_id, job_id=job_id)
-
- url = f"https://ml.googleapis.com/v1/projects/{project_id}/jobs/{job_id}"
- return await self._get_link(url=url, session=session)
-
- async def get_job_status(
- self,
- job_id: str,
- project_id: str = PROVIDE_PROJECT_ID,
- ) -> str | None:
- """
- Poll for job status asynchronously using gcloud-aio.
-
- Note that an OSError is raised when Job results are still pending.
- Exception means that Job finished with errors
- """
- self._check_fileds(project_id=project_id, job_id=job_id)
- async with ClientSession() as session:
- try:
- job = await self.get_job(
- project_id=project_id,
- job_id=job_id,
- session=session, # type: ignore
- )
- job = await job.json(content_type=None)
- self.log.info("Retrieving json_response: %s", job)
-
- if job["state"] in ["SUCCEEDED", "FAILED", "CANCELLED"]:
- job_status = "success"
- elif job["state"] in ["PREPARING", "RUNNING"]:
- job_status = "pending"
- except OSError:
- job_status = "pending"
- except Exception as e:
- self.log.info("Query execution finished with errors...")
- job_status = str(e)
- return job_status
diff --git a/providers/google/src/airflow/providers/google/cloud/links/mlengine.py b/providers/google/src/airflow/providers/google/cloud/links/mlengine.py
deleted file mode 100644
index 040a4181020..00000000000
--- a/providers/google/src/airflow/providers/google/cloud/links/mlengine.py
+++ /dev/null
@@ -1,71 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""This module contains Google ML Engine links."""
-
-from __future__ import annotations
-
-from airflow.providers.google.cloud.links.base import BaseGoogleLink
-
-MLENGINE_BASE_LINK = "https://console.cloud.google.com/ai-platform"
-MLENGINE_MODEL_DETAILS_LINK = MLENGINE_BASE_LINK + "/models/{model_id}/versions?project={project_id}"
-MLENGINE_MODEL_VERSION_DETAILS_LINK = (
- MLENGINE_BASE_LINK + "/models/{model_id}/versions/{version_id}/performance?project={project_id}"
-)
-MLENGINE_MODELS_LIST_LINK = MLENGINE_BASE_LINK + "/models/?project={project_id}"
-MLENGINE_JOB_DETAILS_LINK = MLENGINE_BASE_LINK + "/jobs/{job_id}?project={project_id}"
-MLENGINE_JOBS_LIST_LINK = MLENGINE_BASE_LINK + "/jobs?project={project_id}"
-
-
-class MLEngineModelLink(BaseGoogleLink):
- """Helper class for constructing ML Engine link."""
-
- name = "MLEngine Model"
- key = "ml_engine_model"
- format_str = MLENGINE_MODEL_DETAILS_LINK
-
-
-class MLEngineModelsListLink(BaseGoogleLink):
- """Helper class for constructing ML Engine link."""
-
- name = "MLEngine Models List"
- key = "ml_engine_models_list"
- format_str = MLENGINE_MODELS_LIST_LINK
-
-
-class MLEngineJobDetailsLink(BaseGoogleLink):
- """Helper class for constructing ML Engine link."""
-
- name = "MLEngine Job Details"
- key = "ml_engine_job_details"
- format_str = MLENGINE_JOB_DETAILS_LINK
-
-
-class MLEngineModelVersionDetailsLink(BaseGoogleLink):
- """Helper class for constructing ML Engine link."""
-
- name = "MLEngine Version Details"
- key = "ml_engine_version_details"
- format_str = MLENGINE_MODEL_VERSION_DETAILS_LINK
-
-
-class MLEngineJobSListLink(BaseGoogleLink):
- """Helper class for constructing ML Engine link."""
-
- name = "MLEngine Jobs List"
- key = "ml_engine_jobs_list"
- format_str = MLENGINE_JOBS_LIST_LINK
diff --git a/providers/google/src/airflow/providers/google/cloud/triggers/mlengine.py b/providers/google/src/airflow/providers/google/cloud/triggers/mlengine.py
deleted file mode 100644
index 3ce8d25aa18..00000000000
--- a/providers/google/src/airflow/providers/google/cloud/triggers/mlengine.py
+++ /dev/null
@@ -1,125 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-from __future__ import annotations
-
-import asyncio
-from collections.abc import AsyncIterator, Sequence
-from typing import Any
-
-from airflow.providers.google.cloud.hooks.mlengine import MLEngineAsyncHook
-from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
-from airflow.triggers.base import BaseTrigger, TriggerEvent
-
-
-class MLEngineStartTrainingJobTrigger(BaseTrigger):
- """
- MLEngineStartTrainingJobTrigger run on the trigger worker to perform starting training job operation.
-
- :param conn_id: Reference to google cloud connection id
- :param job_id: The ID of the job. It will be suffixed with hash of job configuration
- :param project_id: Google Cloud Project where the job is running
- :param poll_interval: polling period in seconds to check for the status
- """
-
- def __init__(
- self,
- conn_id: str,
- job_id: str,
- region: str,
- poll_interval: float = 4.0,
- package_uris: list[str] | None = None,
- training_python_module: str | None = None,
- training_args: list[str] | None = None,
- runtime_version: str | None = None,
- python_version: str | None = None,
- job_dir: str | None = None,
- project_id: str = PROVIDE_PROJECT_ID,
- labels: dict[str, str] | None = None,
- gcp_conn_id: str = "google_cloud_default",
- impersonation_chain: str | Sequence[str] | None = None,
- ):
- super().__init__()
- self.log.info("Using the connection %s .", conn_id)
- self.conn_id = conn_id
- self.job_id = job_id
- self._job_conn = None
- self.project_id = project_id
- self.region = region
- self.poll_interval = poll_interval
- self.runtime_version = runtime_version
- self.python_version = python_version
- self.job_dir = job_dir
- self.package_uris = package_uris
- self.training_python_module = training_python_module
- self.training_args = training_args
- self.labels = labels
- self.gcp_conn_id = gcp_conn_id
- self.impersonation_chain = impersonation_chain
-
- def serialize(self) -> tuple[str, dict[str, Any]]:
- """Serialize MLEngineStartTrainingJobTrigger arguments and classpath."""
- return (
- "airflow.providers.google.cloud.triggers.mlengine.MLEngineStartTrainingJobTrigger",
- {
- "conn_id": self.conn_id,
- "job_id": self.job_id,
- "poll_interval": self.poll_interval,
- "region": self.region,
- "project_id": self.project_id,
- "runtime_version": self.runtime_version,
- "python_version": self.python_version,
- "job_dir": self.job_dir,
- "package_uris": self.package_uris,
- "training_python_module": self.training_python_module,
- "training_args": self.training_args,
- "labels": self.labels,
- },
- )
-
- async def run(self) -> AsyncIterator[TriggerEvent]:
- """Get current job execution status and yields a TriggerEvent."""
- hook = self._get_async_hook()
- try:
- while True:
- # Poll for job execution status
- response_from_hook = await hook.get_job_status(job_id=self.job_id, project_id=self.project_id)
- if response_from_hook == "success":
- yield TriggerEvent(
- {
- "job_id": self.job_id,
- "status": "success",
- "message": "Job completed",
- }
- )
- return
- elif response_from_hook == "pending":
- self.log.info("Job is still running...")
- self.log.info("Sleeping for %s seconds.", self.poll_interval)
- await asyncio.sleep(self.poll_interval)
- else:
- yield TriggerEvent({"status": "error", "message": response_from_hook})
- return
-
- except Exception as e:
- self.log.exception("Exception occurred while checking for query completion")
- yield TriggerEvent({"status": "error", "message": str(e)})
-
- def _get_async_hook(self) -> MLEngineAsyncHook:
- return MLEngineAsyncHook(
- gcp_conn_id=self.conn_id,
- impersonation_chain=self.impersonation_chain,
- )
diff --git a/providers/google/src/airflow/providers/google/get_provider_info.py b/providers/google/src/airflow/providers/google/get_provider_info.py
index 5fa1e538c74..d0122a94c51 100644
--- a/providers/google/src/airflow/providers/google/get_provider_info.py
+++ b/providers/google/src/airflow/providers/google/get_provider_info.py
@@ -411,13 +411,6 @@ def get_provider_info():
"logo": "/docs/integration-logos/Kubernetes-Engine.png",
"tags": ["gcp"],
},
- {
- "integration-name": "Google Machine Learning Engine",
- "external-doc-url": "https://cloud.google.com/vertex-ai/",
- "how-to-guide": ["/docs/apache-airflow-providers-google/operators/cloud/mlengine.rst"],
- "logo": "/docs/integration-logos/AI-Platform.png",
- "tags": ["gcp"],
- },
{
"integration-name": "Google Cloud Natural Language",
"external-doc-url": "https://cloud.google.com/natural-language/",
@@ -933,10 +926,6 @@ def get_provider_info():
"integration-name": "Google Kubernetes Engine",
"python-modules": ["airflow.providers.google.cloud.hooks.kubernetes_engine"],
},
- {
- "integration-name": "Google Machine Learning Engine",
- "python-modules": ["airflow.providers.google.cloud.hooks.mlengine"],
- },
{
"integration-name": "Google Cloud Natural Language",
"python-modules": ["airflow.providers.google.cloud.hooks.natural_language"],
@@ -1146,10 +1135,6 @@ def get_provider_info():
"integration-name": "Google Kubernetes Engine",
"python-modules": ["airflow.providers.google.cloud.triggers.kubernetes_engine"],
},
- {
- "integration-name": "Google Machine Learning Engine",
- "python-modules": ["airflow.providers.google.cloud.triggers.mlengine"],
- },
{
"integration-name": "Google Cloud Pub/Sub",
"python-modules": ["airflow.providers.google.cloud.triggers.pubsub"],
@@ -1678,11 +1663,6 @@ def get_provider_info():
"airflow.providers.google.cloud.links.data_loss_prevention.CloudDLPInfoTypesListLink",
"airflow.providers.google.cloud.links.data_loss_prevention.CloudDLPInfoTypeDetailsLink",
"airflow.providers.google.cloud.links.data_loss_prevention.CloudDLPPossibleInfoTypesListLink",
- "airflow.providers.google.cloud.links.mlengine.MLEngineModelLink",
- "airflow.providers.google.cloud.links.mlengine.MLEngineModelsListLink",
- "airflow.providers.google.cloud.links.mlengine.MLEngineJobDetailsLink",
- "airflow.providers.google.cloud.links.mlengine.MLEngineJobSListLink",
- "airflow.providers.google.cloud.links.mlengine.MLEngineModelVersionDetailsLink",
"airflow.providers.google.common.links.storage.StorageLink",
"airflow.providers.google.common.links.storage.FileDetailsLink",
"airflow.providers.google.marketing_platform.links.analytics_admin.GoogleAnalyticsPropertyLink",
diff --git a/providers/google/tests/unit/google/cloud/hooks/test_mlengine.py b/providers/google/tests/unit/google/cloud/hooks/test_mlengine.py
deleted file mode 100644
index 8770138e27b..00000000000
--- a/providers/google/tests/unit/google/cloud/hooks/test_mlengine.py
+++ /dev/null
@@ -1,1128 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-from __future__ import annotations
-
-import json
-import logging
-from copy import deepcopy
-from unittest import mock
-from unittest.mock import PropertyMock
-
-import httplib2
-import pytest
-from aiohttp import ClientResponse
-from aiohttp.helpers import TimerNoop
-from googleapiclient.errors import HttpError
-from yarl import URL
-
-from airflow.providers.common.compat.sdk import AirflowException
-from airflow.providers.google.cloud.hooks import mlengine as hook
-from airflow.providers.google.cloud.hooks.mlengine import MLEngineAsyncHook
-
-from unit.google.cloud.utils.base_gcp_mock import (
- GCP_PROJECT_ID_HOOK_UNIT_TEST,
- mock_base_gcp_hook_default_project_id,
-)
-
-mlengine_hook = MLEngineAsyncHook()
-PROJECT_ID = "test-project"
-JOB_ID = "test-job-id"
-
-
-@pytest.mark.db_test
-class TestMLEngineHook:
- def setup_method(self):
- self.hook = hook.MLEngineHook()
-
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineHook._authorize")
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.build")
- def test_mle_engine_client_creation(self, mock_build, mock_authorize):
- result = self.hook.get_conn()
-
- assert mock_build.return_value == result
- mock_build.assert_called_with("ml", "v1", http=mock_authorize.return_value, cache_discovery=False)
-
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineHook.get_conn")
- def test_create_version(self, mock_get_conn):
- project_id = "test-project"
- model_name = "test-model"
- version_name = "test-version"
- version = {"name": version_name, "labels": {"other-label": "test-value"}}
- version_with_airflow_version = {
- "name": "test-version",
- "labels": {"other-label": "test-value", "airflow-version": hook._AIRFLOW_VERSION},
- }
- operation_path = f"projects/{project_id}/operations/test-operation"
- model_path = f"projects/{project_id}/models/{model_name}"
- operation_done = {"name": operation_path, "done": True}
- (
- mock_get_conn.return_value.projects.return_value.models.return_value.versions.return_value.create.return_value.execute.return_value
- ) = version
- (
- mock_get_conn.return_value.projects.return_value.operations.return_value.get.return_value.execute.return_value
- ) = {"name": operation_path, "done": True}
- create_version_response = self.hook.create_version(
- project_id=project_id, model_name=model_name, version_spec=deepcopy(version)
- )
-
- assert create_version_response == operation_done
-
- mock_get_conn.assert_has_calls(
- [
- mock.call()
- .projects()
- .models()
- .versions()
- .create(body=version_with_airflow_version, parent=model_path),
- mock.call().projects().models().versions().create().execute(num_retries=5),
- mock.call().projects().operations().get(name=version_name),
- ],
- any_order=True,
- )
-
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineHook.get_conn")
- def test_create_version_with_labels(self, mock_get_conn):
- project_id = "test-project"
- model_name = "test-model"
- version_name = "test-version"
- version = {"name": version_name}
- version_with_airflow_version = {
- "name": "test-version",
- "labels": {"airflow-version": hook._AIRFLOW_VERSION},
- }
- operation_path = f"projects/{project_id}/operations/test-operation"
- model_path = f"projects/{project_id}/models/{model_name}"
- operation_done = {"name": operation_path, "done": True}
-
- (
- mock_get_conn.return_value.projects.return_value.models.return_value.versions.return_value.create.return_value.execute.return_value
- ) = version
- (
- mock_get_conn.return_value.projects.return_value.operations.return_value.get.return_value.execute.return_value
- ) = {"name": operation_path, "done": True}
-
- create_version_response = self.hook.create_version(
- project_id=project_id, model_name=model_name, version_spec=deepcopy(version)
- )
-
- assert create_version_response == operation_done
-
- mock_get_conn.assert_has_calls(
- [
- mock.call()
- .projects()
- .models()
- .versions()
- .create(body=version_with_airflow_version, parent=model_path),
- mock.call().projects().models().versions().create().execute(num_retries=5),
- mock.call().projects().operations().get(name=version_name),
- ],
- any_order=True,
- )
-
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineHook.get_conn")
- def test_set_default_version(self, mock_get_conn):
- project_id = "test-project"
- model_name = "test-model"
- version_name = "test-version"
- operation_path = f"projects/{project_id}/operations/test-operation"
- version_path = f"projects/{project_id}/models/{model_name}/versions/{version_name}"
- operation_done = {"name": operation_path, "done": True}
-
- (
- mock_get_conn.return_value.projects.return_value.models.return_value.versions.return_value.setDefault.return_value.execute.return_value
- ) = operation_done
-
- set_default_version_response = self.hook.set_default_version(
- project_id=project_id, model_name=model_name, version_name=version_name
- )
-
- assert set_default_version_response == operation_done
-
- mock_get_conn.assert_has_calls(
- [
- mock.call().projects().models().versions().setDefault(body={}, name=version_path),
- mock.call().projects().models().versions().setDefault().execute(num_retries=5),
- ],
- any_order=True,
- )
-
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.time.sleep")
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineHook.get_conn")
- def test_list_versions(self, mock_get_conn, mock_sleep):
- project_id = "test-project"
- model_name = "test-model"
- model_path = f"projects/{project_id}/models/{model_name}"
- version_names = [f"ver_{ix}" for ix in range(3)]
- response_bodies = [
- {"nextPageToken": f"TOKEN-{ix}", "versions": [ver]} for ix, ver in enumerate(version_names)
- ]
- response_bodies[-1].pop("nextPageToken")
-
- pages_requests = [mock.Mock(**{"execute.return_value": body}) for body in response_bodies]
- versions_mock = mock.Mock(
- **{"list.return_value": pages_requests[0], "list_next.side_effect": pages_requests[1:] + [None]}
- )
-
- (
- mock_get_conn.return_value.projects.return_value.models.return_value.versions.return_value
- ) = versions_mock
-
- list_versions_response = self.hook.list_versions(project_id=project_id, model_name=model_name)
-
- assert list_versions_response == version_names
- mock_get_conn.assert_has_calls(
- [
- mock.call().projects().models().versions().list(pageSize=100, parent=model_path),
- mock.call().projects().models().versions().list().execute(num_retries=5),
- ]
- + [
- mock.call()
- .projects()
- .models()
- .versions()
- .list_next(previous_request=pages_requests[i], previous_response=response_bodies[i])
- for i in range(3)
- ],
- any_order=True,
- )
-
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineHook.get_conn")
- def test_delete_version(self, mock_get_conn):
- project_id = "test-project"
- model_name = "test-model"
- version_name = "test-version"
- operation_path = f"projects/{project_id}/operations/test-operation"
- version_path = f"projects/{project_id}/models/{model_name}/versions/{version_name}"
- version = {"name": operation_path}
- operation_not_done = {"name": operation_path, "done": False}
- operation_done = {"name": operation_path, "done": True}
-
- (
- mock_get_conn.return_value.projects.return_value.operations.return_value.get.return_value.execute.side_effect
- ) = [operation_not_done, operation_done]
-
- (
- mock_get_conn.return_value.projects.return_value.models.return_value.versions.return_value.delete.return_value.execute.return_value
- ) = version
-
- delete_version_response = self.hook.delete_version(
- project_id=project_id, model_name=model_name, version_name=version_name
- )
-
- assert delete_version_response == operation_done
- mock_get_conn.assert_has_calls(
- [
- mock.call().projects().models().versions().delete(name=version_path),
- mock.call().projects().models().versions().delete().execute(num_retries=5),
- mock.call().projects().operations().get(name=operation_path),
- mock.call().projects().operations().get().execute(num_retries=5),
- ],
- any_order=True,
- )
-
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineHook.get_conn")
- def test_create_model(self, mock_get_conn):
- project_id = "test-project"
- model_name = "test-model"
- model = {
- "name": model_name,
- }
- model_with_airflow_version = {
- "name": model_name,
- "labels": {"airflow-version": hook._AIRFLOW_VERSION},
- }
- project_path = f"projects/{project_id}"
-
- (
- mock_get_conn.return_value.projects.return_value.models.return_value.create.return_value.execute.return_value
- ) = model
-
- create_model_response = self.hook.create_model(project_id=project_id, model=deepcopy(model))
-
- assert create_model_response == model
- mock_get_conn.assert_has_calls(
- [
- mock.call().projects().models().create(body=model_with_airflow_version, parent=project_path),
- mock.call().projects().models().create().execute(num_retries=5),
- ]
- )
-
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineHook.get_conn")
- def test_create_model_idempotency(self, mock_get_conn):
- project_id = "test-project"
- model_name = "test-model"
- model = {
- "name": model_name,
- }
- model_with_airflow_version = {
- "name": model_name,
- "labels": {"airflow-version": hook._AIRFLOW_VERSION},
- }
- project_path = f"projects/{project_id}"
-
- (
- mock_get_conn.return_value.projects.return_value.models.return_value.create.return_value.execute.side_effect
- ) = [
- HttpError(
- resp=httplib2.Response({"status": 409}),
- content=json.dumps(
- {
- "error": {
- "code": 409,
- "message": "Field: model.name Error: A model with the same name already exists.",
- "status": "ALREADY_EXISTS",
- "details": [
- {
- "@type": "type.googleapis.com/google.rpc.BadRequest",
- "fieldViolations": [
- {
- "field": "model.name",
- "description": "A model with the same name already exists.",
- }
- ],
- }
- ],
- }
- }
- ).encode(),
- )
- ]
-
- (
- mock_get_conn.return_value.projects.return_value.models.return_value.get.return_value.execute.return_value
- ) = deepcopy(model)
-
- create_model_response = self.hook.create_model(project_id=project_id, model=deepcopy(model))
-
- assert create_model_response == model
- mock_get_conn.assert_has_calls(
- [
- mock.call().projects().models().create(body=model_with_airflow_version, parent=project_path),
- mock.call().projects().models().create().execute(num_retries=5),
- ]
- )
- mock_get_conn.assert_has_calls(
- [
- mock.call().projects().models().get(name="projects/test-project/models/test-model"),
- mock.call().projects().models().get().execute(num_retries=5),
- ]
- )
-
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineHook.get_conn")
- def test_create_model_with_labels(self, mock_get_conn):
- project_id = "test-project"
- model_name = "test-model"
- model = {"name": model_name, "labels": {"other-label": "test-value"}}
- model_with_airflow_version = {
- "name": model_name,
- "labels": {"other-label": "test-value", "airflow-version": hook._AIRFLOW_VERSION},
- }
- project_path = f"projects/{project_id}"
-
- (
- mock_get_conn.return_value.projects.return_value.models.return_value.create.return_value.execute.return_value
- ) = model
-
- create_model_response = self.hook.create_model(project_id=project_id, model=deepcopy(model))
-
- assert create_model_response == model
- mock_get_conn.assert_has_calls(
- [
- mock.call().projects().models().create(body=model_with_airflow_version, parent=project_path),
- mock.call().projects().models().create().execute(num_retries=5),
- ]
- )
-
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineHook.get_conn")
- def test_get_model(self, mock_get_conn):
- project_id = "test-project"
- model_name = "test-model"
- model = {"model": model_name}
- model_path = f"projects/{project_id}/models/{model_name}"
-
- (
- mock_get_conn.return_value.projects.return_value.models.return_value.get.return_value.execute.return_value
- ) = model
-
- get_model_response = self.hook.get_model(project_id=project_id, model_name=model_name)
-
- assert get_model_response == model
- mock_get_conn.assert_has_calls(
- [
- mock.call().projects().models().get(name=model_path),
- mock.call().projects().models().get().execute(num_retries=5),
- ]
- )
-
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineHook.get_conn")
- def test_delete_model(self, mock_get_conn):
- project_id = "test-project"
- model_name = "test-model"
- model = {"model": model_name}
- model_path = f"projects/{project_id}/models/{model_name}"
-
- (
- mock_get_conn.return_value.projects.return_value.models.return_value.delete.return_value.execute.return_value
- ) = model
-
- self.hook.delete_model(project_id=project_id, model_name=model_name)
-
- mock_get_conn.assert_has_calls(
- [
- mock.call().projects().models().delete(name=model_path),
- mock.call().projects().models().delete().execute(num_retries=5),
- ]
- )
-
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineHook.log")
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineHook.get_conn")
- def test_delete_model_when_not_exists(self, mock_get_conn, mock_log):
- project_id = "test-project"
- model_name = "test-model"
- model_path = f"projects/{project_id}/models/{model_name}"
- http_error = HttpError(
- resp=mock.MagicMock(status=404, reason="Model not found."), content=b"Model not found."
- )
-
- (
- mock_get_conn.return_value.projects.return_value.models.return_value.delete.return_value.execute.side_effect
- ) = [http_error]
-
- self.hook.delete_model(project_id=project_id, model_name=model_name)
-
- mock_get_conn.assert_has_calls(
- [
- mock.call().projects().models().delete(name=model_path),
- mock.call().projects().models().delete().execute(num_retries=5),
- ]
- )
- mock_log.error.assert_called_once_with("Model was not found: %s", http_error)
-
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.time.sleep")
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineHook.get_conn")
- def test_delete_model_with_contents(self, mock_get_conn, mock_sleep):
- project_id = "test-project"
- model_name = "test-model"
- model_path = f"projects/{project_id}/models/{model_name}"
- operation_path = f"projects/{project_id}/operations/test-operation"
- operation_done = {"name": operation_path, "done": True}
- version_names = ["AAA", "BBB", "CCC"]
- versions = [
- {
- "name": f"projects/{project_id}/models/{model_name}/versions/{version_name}",
- "isDefault": i == 0,
- }
- for i, version_name in enumerate(version_names)
- ]
-
- (
- mock_get_conn.return_value.projects.return_value.operations.return_value.get.return_value.execute.return_value
- ) = operation_done
- (
- mock_get_conn.return_value.projects.return_value.models.return_value.versions.return_value.list.return_value.execute.return_value
- ) = {"versions": versions}
- (
- mock_get_conn.return_value.projects.return_value.models.return_value.versions.return_value.list_next.return_value
- ) = None
-
- self.hook.delete_model(project_id=project_id, model_name=model_name, delete_contents=True)
-
- mock_get_conn.assert_has_calls(
- [
- mock.call().projects().models().delete(name=model_path),
- mock.call().projects().models().delete().execute(num_retries=5),
- ]
- + [
- mock.call()
- .projects()
- .models()
- .versions()
- .delete(
- name=f"projects/{project_id}/models/{model_name}/versions/{version_name}",
- )
- for version_name in version_names
- ],
- any_order=True,
- )
-
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.time.sleep")
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineHook.get_conn")
- def test_create_mlengine_job(self, mock_get_conn, mock_sleep):
- project_id = "test-project"
- job_id = "test-job-id"
- project_path = f"projects/{project_id}"
- job_path = f"projects/{project_id}/jobs/{job_id}"
- new_job = {
- "jobId": job_id,
- "foo": 4815162342,
- }
- new_job_with_airflow_version = {
- "jobId": job_id,
- "foo": 4815162342,
- "labels": {"airflow-version": hook._AIRFLOW_VERSION},
- }
-
- job_succeeded = {
- "jobId": job_id,
- "state": "SUCCEEDED",
- }
- job_queued = {
- "jobId": job_id,
- "state": "QUEUED",
- }
-
- (
- mock_get_conn.return_value.projects.return_value.jobs.return_value.create.return_value.execute.return_value
- ) = job_queued
- (
- mock_get_conn.return_value.projects.return_value.jobs.return_value.get.return_value.execute.side_effect
- ) = [job_queued, job_succeeded]
-
- create_job_response = self.hook.create_job(project_id=project_id, job=deepcopy(new_job))
-
- assert create_job_response == job_succeeded
- mock_get_conn.assert_has_calls(
- [
- mock.call().projects().jobs().create(body=new_job_with_airflow_version, parent=project_path),
- mock.call().projects().jobs().get(name=job_path),
- mock.call().projects().jobs().get().execute(num_retries=5),
- ],
- any_order=True,
- )
-
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.time.sleep")
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineHook.get_conn")
- def test_create_mlengine_job_with_labels(self, mock_get_conn, mock_sleep):
- project_id = "test-project"
- job_id = "test-job-id"
- project_path = f"projects/{project_id}"
- job_path = f"projects/{project_id}/jobs/{job_id}"
- new_job = {"jobId": job_id, "foo": 4815162342, "labels": {"other-label": "test-value"}}
- new_job_with_airflow_version = {
- "jobId": job_id,
- "foo": 4815162342,
- "labels": {"other-label": "test-value", "airflow-version": hook._AIRFLOW_VERSION},
- }
-
- job_succeeded = {
- "jobId": job_id,
- "state": "SUCCEEDED",
- }
- job_queued = {
- "jobId": job_id,
- "state": "QUEUED",
- }
-
- (
- mock_get_conn.return_value.projects.return_value.jobs.return_value.create.return_value.execute.return_value
- ) = job_queued
- (
- mock_get_conn.return_value.projects.return_value.jobs.return_value.get.return_value.execute.side_effect
- ) = [job_queued, job_succeeded]
-
- create_job_response = self.hook.create_job(project_id=project_id, job=deepcopy(new_job))
-
- assert create_job_response == job_succeeded
- mock_get_conn.assert_has_calls(
- [
- mock.call().projects().jobs().create(body=new_job_with_airflow_version, parent=project_path),
- mock.call().projects().jobs().get(name=job_path),
- mock.call().projects().jobs().get().execute(num_retries=5),
- ],
- any_order=True,
- )
-
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineHook.get_conn")
- def test_create_mlengine_job_reuse_existing_job_by_default(self, mock_get_conn):
- project_id = "test-project"
- job_id = "test-job-id"
- project_path = f"projects/{project_id}"
- job_path = f"projects/{project_id}/jobs/{job_id}"
- job_succeeded = {
- "jobId": job_id,
- "foo": 4815162342,
- "state": "SUCCEEDED",
- }
- error_job_exists = HttpError(resp=mock.MagicMock(status=409), content=b"Job already exists")
-
- (
- mock_get_conn.return_value.projects.return_value.jobs.return_value.create.return_value.execute.side_effect
- ) = error_job_exists
- (
- mock_get_conn.return_value.projects.return_value.jobs.return_value.get.return_value.execute.return_value
- ) = job_succeeded
-
- create_job_response = self.hook.create_job(project_id=project_id, job=job_succeeded)
-
- assert create_job_response == job_succeeded
- mock_get_conn.assert_has_calls(
- [
- mock.call().projects().jobs().create(body=job_succeeded, parent=project_path),
- mock.call().projects().jobs().create().execute(num_retries=5),
- mock.call().projects().jobs().get(name=job_path),
- mock.call().projects().jobs().get().execute(num_retries=5),
- ],
- any_order=True,
- )
-
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineHook.get_conn")
- def test_create_mlengine_job_check_existing_job_failed(self, mock_get_conn):
- project_id = "test-project"
- job_id = "test-job-id"
- my_job = {
- "jobId": job_id,
- "foo": 4815162342,
- "state": "SUCCEEDED",
- "someInput": {"input": "someInput"},
- }
- different_job = {
- "jobId": job_id,
- "foo": 4815162342,
- "state": "SUCCEEDED",
- "someInput": {"input": "someDifferentInput"},
- }
- error_job_exists = HttpError(resp=mock.MagicMock(status=409), content=b"Job already exists")
-
- (
- mock_get_conn.return_value.projects.return_value.jobs.return_value.create.return_value.execute.side_effect
- ) = error_job_exists
- (
- mock_get_conn.return_value.projects.return_value.jobs.return_value.get.return_value.execute.return_value
- ) = different_job
-
- def check_input(existing_job):
- return existing_job.get("someInput") == my_job["someInput"]
-
- with pytest.raises(HttpError):
- self.hook.create_job(project_id=project_id, job=my_job, use_existing_job_fn=check_input)
-
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineHook.get_conn")
- def test_create_mlengine_job_check_existing_job_success(self, mock_get_conn):
- project_id = "test-project"
- job_id = "test-job-id"
- my_job = {
- "jobId": job_id,
- "foo": 4815162342,
- "state": "SUCCEEDED",
- "someInput": {"input": "someInput"},
- }
- error_job_exists = HttpError(resp=mock.MagicMock(status=409), content=b"Job already exists")
-
- (
- mock_get_conn.return_value.projects.return_value.jobs.return_value.create.return_value.execute.side_effect
- ) = error_job_exists
- (
- mock_get_conn.return_value.projects.return_value.jobs.return_value.get.return_value.execute.return_value
- ) = my_job
-
- def check_input(existing_job):
- return existing_job.get("someInput") == my_job["someInput"]
-
- create_job_response = self.hook.create_job(
- project_id=project_id, job=my_job, use_existing_job_fn=check_input
- )
-
- assert create_job_response == my_job
-
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineHook.get_conn")
- def test_cancel_mlengine_job(self, mock_get_conn):
- project_id = "test-project"
- job_id = "test-job-id"
- job_path = f"projects/{project_id}/jobs/{job_id}"
-
- job_cancelled = {}
-
- (
- mock_get_conn.return_value.projects.return_value.jobs.return_value.cancel.return_value.execute.return_value
- ) = job_cancelled
-
- cancel_job_response = self.hook.cancel_job(job_id=job_id, project_id=project_id)
-
- assert cancel_job_response == job_cancelled
- mock_get_conn.assert_has_calls(
- [
- mock.call().projects().jobs().cancel(name=job_path),
- mock.call().projects().jobs().cancel().execute(num_retries=5),
- ],
- any_order=True,
- )
-
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineHook.get_conn")
- def test_cancel_mlengine_job_nonexistent_job(self, mock_get_conn):
- project_id = "test-project"
- job_id = "test-job-id"
- job_cancelled = {}
-
- error_job_does_not_exist = HttpError(resp=mock.MagicMock(status=404), content=b"Job does not exist")
-
- (
- mock_get_conn.return_value.projects.return_value.jobs.return_value.cancel.return_value.execute.side_effect
- ) = error_job_does_not_exist
- (
- mock_get_conn.return_value.projects.return_value.jobs.return_value.cancel.return_value.execute.return_value
- ) = job_cancelled
-
- with pytest.raises(HttpError):
- self.hook.cancel_job(job_id=job_id, project_id=project_id)
-
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineHook.get_conn")
- def test_cancel_mlengine_job_completed_job(self, mock_get_conn):
- project_id = "test-project"
- job_id = "test-job-id"
- job_path = f"projects/{project_id}/jobs/{job_id}"
- job_cancelled = {}
-
- error_job_already_completed = HttpError(
- resp=mock.MagicMock(status=400), content=b"Job already completed"
- )
-
- (
- mock_get_conn.return_value.projects.return_value.jobs.return_value.cancel.return_value.execute.side_effect
- ) = error_job_already_completed
- (
- mock_get_conn.return_value.projects.return_value.jobs.return_value.cancel.return_value.execute.return_value
- ) = job_cancelled
-
- cancel_job_response = self.hook.cancel_job(job_id=job_id, project_id=project_id)
-
- assert cancel_job_response == job_cancelled
- mock_get_conn.assert_has_calls(
- [
- mock.call().projects().jobs().cancel(name=job_path),
- mock.call().projects().jobs().cancel().execute(num_retries=5),
- ],
- any_order=True,
- )
-
-
-class TestMLEngineHookWithDefaultProjectId:
- def setup_method(self):
- with mock.patch(
- "airflow.providers.google.cloud.hooks.mlengine.MLEngineHook.__init__",
- new=mock_base_gcp_hook_default_project_id,
- ):
- self.hook = hook.MLEngineHook()
-
- @mock.patch(
- "airflow.providers.google.common.hooks.base_google.GoogleBaseHook.project_id",
- new_callable=PropertyMock,
- return_value=GCP_PROJECT_ID_HOOK_UNIT_TEST,
- )
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineHook.get_conn")
- def test_create_version(self, mock_get_conn, mock_project_id):
- model_name = "test-model"
- version_name = "test-version"
- version = {"name": version_name}
- operation_path = f"projects/{GCP_PROJECT_ID_HOOK_UNIT_TEST}/operations/test-operation"
- model_path = f"projects/{GCP_PROJECT_ID_HOOK_UNIT_TEST}/models/{model_name}"
- operation_done = {"name": operation_path, "done": True}
-
- (
- mock_get_conn.return_value.projects.return_value.models.return_value.versions.return_value.create.return_value.execute.return_value
- ) = version
- (
- mock_get_conn.return_value.projects.return_value.operations.return_value.get.return_value.execute.return_value
- ) = {"name": operation_path, "done": True}
-
- create_version_response = self.hook.create_version(
- model_name=model_name, version_spec=version, project_id=GCP_PROJECT_ID_HOOK_UNIT_TEST
- )
-
- assert create_version_response == operation_done
- mock_get_conn.assert_has_calls(
- [
- mock.call().projects().models().versions().create(body=version, parent=model_path),
- mock.call().projects().models().versions().create().execute(num_retries=5),
- mock.call().projects().operations().get(name=version_name),
- mock.call().projects().operations().get().execute(num_retries=5),
- ],
- any_order=True,
- )
-
- @mock.patch(
- "airflow.providers.google.common.hooks.base_google.GoogleBaseHook.project_id",
- new_callable=PropertyMock,
- return_value=GCP_PROJECT_ID_HOOK_UNIT_TEST,
- )
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineHook.get_conn")
- def test_set_default_version(self, mock_get_conn, mock_project_id):
- model_name = "test-model"
- version_name = "test-version"
- operation_path = f"projects/{GCP_PROJECT_ID_HOOK_UNIT_TEST}/operations/test-operation"
- version_path = f"projects/{GCP_PROJECT_ID_HOOK_UNIT_TEST}/models/{model_name}/versions/{version_name}"
- operation_done = {"name": operation_path, "done": True}
-
- (
- mock_get_conn.return_value.projects.return_value.models.return_value.versions.return_value.setDefault.return_value.execute.return_value
- ) = operation_done
-
- set_default_version_response = self.hook.set_default_version(
- model_name=model_name,
- version_name=version_name,
- project_id=GCP_PROJECT_ID_HOOK_UNIT_TEST,
- )
-
- assert set_default_version_response == operation_done
- mock_get_conn.assert_has_calls(
- [
- mock.call().projects().models().versions().setDefault(body={}, name=version_path),
- mock.call().projects().models().versions().setDefault().execute(num_retries=5),
- ],
- any_order=True,
- )
-
- @mock.patch(
- "airflow.providers.google.common.hooks.base_google.GoogleBaseHook.project_id",
- new_callable=PropertyMock,
- return_value=GCP_PROJECT_ID_HOOK_UNIT_TEST,
- )
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.time.sleep")
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineHook.get_conn")
- def test_list_versions(self, mock_get_conn, mock_sleep, mock_project_id):
- model_name = "test-model"
- model_path = f"projects/{GCP_PROJECT_ID_HOOK_UNIT_TEST}/models/{model_name}"
- version_names = [f"ver_{ix}" for ix in range(3)]
- response_bodies = [
- {"nextPageToken": f"TOKEN-{ix}", "versions": [ver]} for ix, ver in enumerate(version_names)
- ]
- response_bodies[-1].pop("nextPageToken")
-
- pages_requests = [mock.Mock(**{"execute.return_value": body}) for body in response_bodies]
- versions_mock = mock.Mock(
- **{"list.return_value": pages_requests[0], "list_next.side_effect": pages_requests[1:] + [None]}
- )
-
- (
- mock_get_conn.return_value.projects.return_value.models.return_value.versions.return_value
- ) = versions_mock
-
- list_versions_response = self.hook.list_versions(
- model_name=model_name, project_id=GCP_PROJECT_ID_HOOK_UNIT_TEST
- )
-
- assert list_versions_response == version_names
- mock_get_conn.assert_has_calls(
- [
- mock.call().projects().models().versions().list(pageSize=100, parent=model_path),
- mock.call().projects().models().versions().list().execute(num_retries=5),
- ]
- + [
- mock.call()
- .projects()
- .models()
- .versions()
- .list_next(previous_request=pages_requests[i], previous_response=response_bodies[i])
- for i in range(3)
- ],
- any_order=True,
- )
-
- @mock.patch(
- "airflow.providers.google.common.hooks.base_google.GoogleBaseHook.project_id",
- new_callable=PropertyMock,
- return_value=GCP_PROJECT_ID_HOOK_UNIT_TEST,
- )
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineHook.get_conn")
- def test_delete_version(self, mock_get_conn, mock_project_id):
- model_name = "test-model"
- version_name = "test-version"
-
- operation_path = f"projects/{GCP_PROJECT_ID_HOOK_UNIT_TEST}/operations/test-operation"
- version_path = f"projects/{GCP_PROJECT_ID_HOOK_UNIT_TEST}/models/{model_name}/versions/{version_name}"
- version = {"name": operation_path}
- operation_not_done = {"name": operation_path, "done": False}
- operation_done = {"name": operation_path, "done": True}
-
- (
- mock_get_conn.return_value.projects.return_value.operations.return_value.get.return_value.execute.side_effect
- ) = [operation_not_done, operation_done]
-
- (
- mock_get_conn.return_value.projects.return_value.models.return_value.versions.return_value.delete.return_value.execute.return_value
- ) = version
-
- delete_version_response = self.hook.delete_version(
- model_name=model_name, version_name=version_name, project_id=GCP_PROJECT_ID_HOOK_UNIT_TEST
- )
-
- assert delete_version_response == operation_done
- mock_get_conn.assert_has_calls(
- [
- mock.call().projects().models().versions().delete(name=version_path),
- mock.call().projects().models().versions().delete().execute(num_retries=5),
- mock.call().projects().operations().get(name=operation_path),
- mock.call().projects().operations().get().execute(num_retries=5),
- ],
- any_order=True,
- )
-
- @mock.patch(
- "airflow.providers.google.common.hooks.base_google.GoogleBaseHook.project_id",
- new_callable=PropertyMock,
- return_value=GCP_PROJECT_ID_HOOK_UNIT_TEST,
- )
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineHook.get_conn")
- def test_create_model(self, mock_get_conn, mock_project_id):
- model_name = "test-model"
- model = {
- "name": model_name,
- }
- project_path = f"projects/{GCP_PROJECT_ID_HOOK_UNIT_TEST}"
-
- (
- mock_get_conn.return_value.projects.return_value.models.return_value.create.return_value.execute.return_value
- ) = model
-
- create_model_response = self.hook.create_model(model=model, project_id=GCP_PROJECT_ID_HOOK_UNIT_TEST)
-
- assert create_model_response == model
- mock_get_conn.assert_has_calls(
- [
- mock.call().projects().models().create(body=model, parent=project_path),
- mock.call().projects().models().create().execute(num_retries=5),
- ]
- )
-
- @mock.patch(
- "airflow.providers.google.common.hooks.base_google.GoogleBaseHook.project_id",
- new_callable=PropertyMock,
- return_value=GCP_PROJECT_ID_HOOK_UNIT_TEST,
- )
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineHook.get_conn")
- def test_get_model(self, mock_get_conn, mock_project_id):
- model_name = "test-model"
- model = {"model": model_name}
- model_path = f"projects/{GCP_PROJECT_ID_HOOK_UNIT_TEST}/models/{model_name}"
-
- (
- mock_get_conn.return_value.projects.return_value.models.return_value.get.return_value.execute.return_value
- ) = model
-
- get_model_response = self.hook.get_model(
- model_name=model_name, project_id=GCP_PROJECT_ID_HOOK_UNIT_TEST
- )
-
- assert get_model_response == model
- mock_get_conn.assert_has_calls(
- [
- mock.call().projects().models().get(name=model_path),
- mock.call().projects().models().get().execute(num_retries=5),
- ]
- )
-
- @mock.patch(
- "airflow.providers.google.common.hooks.base_google.GoogleBaseHook.project_id",
- new_callable=PropertyMock,
- return_value=GCP_PROJECT_ID_HOOK_UNIT_TEST,
- )
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineHook.get_conn")
- def test_delete_model(self, mock_get_conn, mock_project_id):
- model_name = "test-model"
- model = {"model": model_name}
- model_path = f"projects/{GCP_PROJECT_ID_HOOK_UNIT_TEST}/models/{model_name}"
-
- (
- mock_get_conn.return_value.projects.return_value.models.return_value.delete.return_value.execute.return_value
- ) = model
-
- self.hook.delete_model(model_name=model_name, project_id=GCP_PROJECT_ID_HOOK_UNIT_TEST)
-
- mock_get_conn.assert_has_calls(
- [
- mock.call().projects().models().delete(name=model_path),
- mock.call().projects().models().delete().execute(num_retries=5),
- ]
- )
-
- @mock.patch(
- "airflow.providers.google.common.hooks.base_google.GoogleBaseHook.project_id",
- new_callable=PropertyMock,
- return_value=GCP_PROJECT_ID_HOOK_UNIT_TEST,
- )
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.time.sleep")
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineHook.get_conn")
- def test_create_mlengine_job(self, mock_get_conn, mock_sleep, mock_project_id):
- job_id = "test-job-id"
- project_path = f"projects/{GCP_PROJECT_ID_HOOK_UNIT_TEST}"
- job_path = f"projects/{GCP_PROJECT_ID_HOOK_UNIT_TEST}/jobs/{job_id}"
- new_job = {
- "jobId": job_id,
- "foo": 4815162342,
- }
- job_succeeded = {
- "jobId": job_id,
- "state": "SUCCEEDED",
- }
- job_queued = {
- "jobId": job_id,
- "state": "QUEUED",
- }
-
- (
- mock_get_conn.return_value.projects.return_value.jobs.return_value.create.return_value.execute.return_value
- ) = job_queued
- (
- mock_get_conn.return_value.projects.return_value.jobs.return_value.get.return_value.execute.side_effect
- ) = [job_queued, job_succeeded]
-
- create_job_response = self.hook.create_job(job=new_job, project_id=GCP_PROJECT_ID_HOOK_UNIT_TEST)
-
- assert create_job_response == job_succeeded
- mock_get_conn.assert_has_calls(
- [
- mock.call().projects().jobs().create(body=new_job, parent=project_path),
- mock.call().projects().jobs().get(name=job_path),
- mock.call().projects().jobs().get().execute(num_retries=5),
- ],
- any_order=True,
- )
-
- @mock.patch(
- "airflow.providers.google.common.hooks.base_google.GoogleBaseHook.project_id",
- new_callable=PropertyMock,
- return_value=GCP_PROJECT_ID_HOOK_UNIT_TEST,
- )
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineHook.get_conn")
- def test_cancel_mlengine_job(self, mock_get_conn, mock_project_id):
- job_id = "test-job-id"
- job_path = f"projects/{GCP_PROJECT_ID_HOOK_UNIT_TEST}/jobs/{job_id}"
-
- job_cancelled = {}
-
- (
- mock_get_conn.return_value.projects.return_value.jobs.return_value.cancel.return_value.execute.return_value
- ) = job_cancelled
-
- cancel_job_response = self.hook.cancel_job(job_id=job_id, project_id=GCP_PROJECT_ID_HOOK_UNIT_TEST)
-
- assert cancel_job_response == job_cancelled
- mock_get_conn.assert_has_calls(
- [
- mock.call().projects().jobs().cancel(name=job_path),
- mock.call().projects().jobs().cancel().execute(num_retries=5),
- ],
- any_order=True,
- )
-
-
-def session():
- return mock.Mock()
-
-
-@pytest.mark.asyncio
-@mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineAsyncHook._get_link")
-async def test_async_get_job_should_execute_successfully(mocked_link):
- await mlengine_hook.get_job(project_id=PROJECT_ID, job_id=JOB_ID, session=session)
- mocked_link.assert_awaited_once_with(
- url=f"https://ml.googleapis.com/v1/projects/{PROJECT_ID}/jobs/{JOB_ID}", session=session
- )
-
-
-@pytest.mark.asyncio
-async def test_async_get_job_should_fail_if_no_job_id():
- with pytest.raises(
- AirflowException, match=r"An unique job id is required for Google MLEngine training job."
- ):
- await mlengine_hook.get_job(project_id=PROJECT_ID, job_id=None, session=session)
-
-
-@pytest.mark.asyncio
-async def test_async_get_job_should_fail_if_no_project_id():
- with pytest.raises(AirflowException, match=r"Google Cloud project id is required."):
- await mlengine_hook.get_job(project_id=None, job_id=JOB_ID, session=session)
-
-
-@pytest.mark.asyncio
-@mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineAsyncHook.get_job")
-async def test_async_get_job_status_should_execute_successfully(mocked_get):
- mocked_get.return_value = ClientResponse(
- "get",
- URL(f"https://ml.googleapis.com/v1/projects/{PROJECT_ID}/jobs/{JOB_ID}"),
- request_info=mock.Mock(),
- writer=mock.Mock(),
- continue100=None,
- timer=TimerNoop(),
- traces=[],
- loop=mock.Mock(),
- session=None,
- )
- mocked_get.return_value._headers = {"Content-Type": "application/json;charset=cp1251"}
- mocked_get.return_value._body = b'{"state": "SUCCEEDED"}'
-
- job_status = await mlengine_hook.get_job_status(job_id=JOB_ID, project_id=PROJECT_ID)
- mocked_get.assert_awaited_once()
- assert job_status == "success"
-
-
-@pytest.mark.asyncio
-@mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineAsyncHook.get_job")
-async def test_async_get_job_status_still_running_should_execute_successfully(mocked_get):
- """Assets that the MLEngineAsyncHook returns a pending response when job is still in running state"""
- mocked_get.return_value = ClientResponse(
- "get",
- URL(f"https://ml.googleapis.com/v1/projects/{PROJECT_ID}/jobs/{JOB_ID}"),
- request_info=mock.Mock(),
- writer=mock.Mock(),
- continue100=None,
- timer=TimerNoop(),
- traces=[],
- loop=mock.Mock(),
- session=None,
- )
- mocked_get.return_value._headers = {"Content-Type": "application/json;charset=cp1251"}
- mocked_get.return_value._body = b'{"state": "RUNNING"}'
-
- job_status = await mlengine_hook.get_job_status(job_id=JOB_ID, project_id=PROJECT_ID)
- mocked_get.assert_awaited_once()
- assert job_status == "pending"
-
-
-@pytest.mark.asyncio
-@mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineAsyncHook.get_job")
-async def test_async_get_job_status_with_oserror_should_execute_successfully(mocked_get):
- """Assets that the MLEngineAsyncHook returns a pending response when OSError is raised"""
- mocked_get.side_effect = OSError()
-
- job_status = await mlengine_hook.get_job_status(job_id=JOB_ID, project_id=PROJECT_ID)
- mocked_get.assert_awaited_once()
- assert job_status == "pending"
-
-
-@pytest.mark.asyncio
-@mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineAsyncHook.get_job")
-async def test_async_get_job_status_with_exception_should_execute_successfully(mocked_get, caplog):
- """Assets that the logging is done correctly when MLEngineAsyncHook raises Exception"""
- caplog.set_level(logging.INFO)
- mocked_get.side_effect = Exception()
-
- await mlengine_hook.get_job_status(job_id=JOB_ID, project_id=PROJECT_ID)
- assert "Query execution finished with errors..." in caplog.text
-
-
-@pytest.mark.asyncio
-async def test_async_get_job_status_should_fail_if_no_job_id():
- with pytest.raises(
- AirflowException, match=r"An unique job id is required for Google MLEngine training job."
- ):
- await mlengine_hook.get_job_status(project_id=PROJECT_ID, job_id=None)
-
-
-@pytest.mark.asyncio
-async def test_async_get_job_status_should_fail_if_no_project_id():
- with pytest.raises(AirflowException, match=r"Google Cloud project id is required."):
- await mlengine_hook.get_job_status(project_id=None, job_id=JOB_ID)
diff --git a/providers/google/tests/unit/google/cloud/triggers/test_mlengine.py b/providers/google/tests/unit/google/cloud/triggers/test_mlengine.py
deleted file mode 100644
index 497793ed3cd..00000000000
--- a/providers/google/tests/unit/google/cloud/triggers/test_mlengine.py
+++ /dev/null
@@ -1,142 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-from __future__ import annotations
-
-import asyncio
-import logging
-from unittest import mock
-
-import pytest
-
-from airflow.providers.google.cloud.triggers.mlengine import MLEngineStartTrainingJobTrigger
-from airflow.triggers.base import TriggerEvent
-
-TEST_CONN_ID = "ml_default"
-TEST_JOB_ID = "1234"
-TEST_GCP_PROJECT_ID = "test-project"
-TEST_REGION = "us-central1"
-TEST_RUNTIME_VERSION = "1.15"
-TEST_PYTHON_VERSION = "3.9"
-TEST_JOB_DIR = "gs://example_mlengine_bucket/job-dir"
-TEST_PACKAGE_URIS = ["gs://system-tests-resources/example_gcp_mlengine/trainer-0.1.tar.gz"]
-TEST_TRAINING_PYTHON_MODULE = "trainer.task"
-TEST_TRAINING_ARGS: list[str] = []
-TEST_LABELS = {"job_type": "training", "***-version": "v2-5-0-dev0"}
-TEST_POLL_INTERVAL = 4.0
-
-
-@pytest.fixture
-def trigger():
- return MLEngineStartTrainingJobTrigger(
- conn_id=TEST_CONN_ID,
- job_id=TEST_JOB_ID,
- project_id=TEST_GCP_PROJECT_ID,
- region=TEST_REGION,
- runtime_version=TEST_RUNTIME_VERSION,
- python_version=TEST_PYTHON_VERSION,
- job_dir=TEST_JOB_DIR,
- package_uris=TEST_PACKAGE_URIS,
- training_python_module=TEST_TRAINING_PYTHON_MODULE,
- training_args=TEST_TRAINING_ARGS,
- labels=TEST_LABELS,
- )
-
-
-class TestMLEngineStartTrainingJobTrigger:
- def test_serialize(self, trigger):
- """
- Asserts that the MLEngineStartTrainingJobTrigger correctly serializes its arguments
- and classpath.
- """
-
- classpath, kwargs = trigger.serialize()
- assert classpath == "airflow.providers.google.cloud.triggers.mlengine.MLEngineStartTrainingJobTrigger"
- assert kwargs == {
- "conn_id": TEST_CONN_ID,
- "job_id": TEST_JOB_ID,
- "project_id": TEST_GCP_PROJECT_ID,
- "region": TEST_REGION,
- "runtime_version": TEST_RUNTIME_VERSION,
- "python_version": TEST_PYTHON_VERSION,
- "job_dir": TEST_JOB_DIR,
- "poll_interval": TEST_POLL_INTERVAL,
- "package_uris": TEST_PACKAGE_URIS,
- "training_python_module": TEST_TRAINING_PYTHON_MODULE,
- "training_args": TEST_TRAINING_ARGS,
- "labels": TEST_LABELS,
- }
-
- @pytest.mark.asyncio
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineAsyncHook.get_job_status")
- async def test_trigger_on_success_yield_successfully(self, mock_job_status, trigger):
- """
- Tests the MLEngineStartTrainingJobTrigger only fires once the job execution reaches a successful state
- """
- mock_job_status.return_value = "success"
-
- generator = trigger.run()
- actual = await generator.asend(None)
- assert (
- TriggerEvent({"status": "success", "message": "Job completed", "job_id": TEST_JOB_ID}) == actual
- )
-
- @pytest.mark.asyncio
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineAsyncHook.get_job")
- async def test_trigger_on_running_wait_successfully(self, mocked_get, caplog, trigger):
- """
- Test that MLEngineStartTrainingJobTrigger does not fire while a job is still running.
- """
-
- mocked_get.side_effect = OSError()
- caplog.set_level(logging.INFO)
-
- task = asyncio.create_task(trigger.run().__anext__())
- await asyncio.sleep(0.5)
-
- # TriggerEvent was not returned
- assert task.done() is False
-
- assert "Job is still running..." in caplog.text
- assert f"Sleeping for {TEST_POLL_INTERVAL} seconds." in caplog.text
-
- # Prevents error when task is destroyed while in "pending" state
- asyncio.get_event_loop().stop()
-
- @pytest.mark.asyncio
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineAsyncHook.get_job_status")
- async def test_trigger_on_error_yield_successfully(self, mock_job_status, trigger):
- """
- Test that MLEngineStartTrainingJobTrigger fires the correct event in case of an error.
- """
- # Set the status to a value other than success or pending
- mock_job_status.return_value = "error"
-
- generator = trigger.run()
- actual = await generator.asend(None)
- assert TriggerEvent({"status": "error", "message": "error"}) == actual
-
- @pytest.mark.asyncio
- @mock.patch("airflow.providers.google.cloud.hooks.mlengine.MLEngineAsyncHook.get_job_status")
- async def test_trigger_exec_yield_successfully(self, mock_job_status, trigger):
- """
- Test that MLEngineStartTrainingJobTrigger fires the correct event in case of an error.
- """
- mock_job_status.side_effect = Exception("Test exception")
-
- generator = trigger.run()
- actual = await generator.asend(None)
- assert TriggerEvent({"status": "error", "message": "Test exception"}) == actual