[airflow] branch main updated: Fix AWS system test example_emr_eks (#31692)

[on...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

onikolas pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 9ea53da222 Fix AWS system test example_emr_eks (#31692)
9ea53da222 is described below

commit 9ea53da222f8c04b2b1bd5fb31f01f42d388d383
Author: Vincent <97...@users.noreply.github.com>
AuthorDate: Fri Jun 2 18:23:42 2023 -0400

    Fix AWS system test example_emr_eks (#31692)
---
 .../system/providers/amazon/aws/example_emr_eks.py | 24 +++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/tests/system/providers/amazon/aws/example_emr_eks.py b/tests/system/providers/amazon/aws/example_emr_eks.py
index 75aba68982..53c865a4a1 100644
--- a/tests/system/providers/amazon/aws/example_emr_eks.py
+++ b/tests/system/providers/amazon/aws/example_emr_eks.py
@@ -43,12 +43,14 @@ DAG_ID = "example_emr_eks"
 # Externally fetched variables
 ROLE_ARN_KEY = "ROLE_ARN"
 JOB_ROLE_ARN_KEY = "JOB_ROLE_ARN"
+JOB_ROLE_NAME_KEY = "JOB_ROLE_NAME"
 SUBNETS_KEY = "SUBNETS"
 
 sys_test_context_task = (
     SystemTestContextBuilder()
     .add_variable(ROLE_ARN_KEY)
     .add_variable(JOB_ROLE_ARN_KEY)
+    .add_variable(JOB_ROLE_NAME_KEY)
     .add_variable(SUBNETS_KEY, split_string=True)
     .build()
 )
@@ -110,10 +112,10 @@ def enable_access_emr_on_eks(cluster, ns):
 
 
 @task
-def create_iam_oidc_identity_provider(cluster):
+def create_iam_oidc_identity_provider(cluster_name):
     # Create an IAM OIDC identity provider
     # See https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/setting-up-enable-IAM.html
-    command = f"eksctl utils associate-iam-oidc-provider --cluster {cluster} --approve"
+    command = f"eksctl utils associate-iam-oidc-provider --cluster {cluster_name} --approve"
 
     build = subprocess.Popen(
         command,
@@ -128,8 +130,16 @@ def create_iam_oidc_identity_provider(cluster):
 
 
 @task
-def get_execution_role_name() -> str:
-    return boto3.client("sts").get_caller_identity()["Arn"].split("/")[-2]
+def delete_iam_oidc_identity_provider(cluster_name):
+    oidc_provider_issuer_url = boto3.client("eks").describe_cluster(name=cluster_name,)["cluster"][
+        "identity"
+    ]["oidc"]["issuer"]
+    oidc_provider_issuer_endpoint = oidc_provider_issuer_url.replace("https://", "")
+
+    account_id = boto3.client("sts").get_caller_identity()["Account"]
+    boto3.client("iam").delete_open_id_connect_provider(
+        OpenIDConnectProviderArn=f"arn:aws:iam::{account_id}:oidc-provider/{oidc_provider_issuer_endpoint}"
+    )
 
 
 @task
@@ -190,6 +200,7 @@ with DAG(
     role_arn = test_context[ROLE_ARN_KEY]
     subnets = test_context[SUBNETS_KEY]
     job_role_arn = test_context[JOB_ROLE_ARN_KEY]
+    job_role_name = test_context[JOB_ROLE_NAME_KEY]
 
     s3_bucket_name = f"{env_id}-bucket"
     eks_cluster_name = f"{env_id}-cluster"
@@ -306,7 +317,6 @@ with DAG(
     chain(
         # TEST SETUP
         test_context,
-        create_launch_template(launch_template_name),
         create_bucket,
         upload_s3_file,
         create_launch_template(launch_template_name),
@@ -314,18 +324,18 @@ with DAG(
         await_create_nodegroup,
         enable_access_emr_on_eks(eks_cluster_name, eks_namespace),
         create_iam_oidc_identity_provider(eks_cluster_name),
-        update_trust_policy_execution_role(eks_cluster_name, eks_namespace, get_execution_role_name()),
+        update_trust_policy_execution_role(eks_cluster_name, eks_namespace, job_role_name),
         # TEST BODY
         create_emr_eks_cluster,
         job_starter,
         job_waiter,
         # TEST TEARDOWN
+        delete_iam_oidc_identity_provider(eks_cluster_name),
         delete_virtual_cluster(str(create_emr_eks_cluster.output)),
         delete_eks_cluster,
         await_delete_eks_cluster,
         delete_launch_template(launch_template_name),
         delete_bucket,
-        delete_launch_template(launch_template_name),
     )
 
     from tests.system.utils.watcher import watcher