You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ks...@apache.org on 2022/05/06 18:20:45 UTC
[arrow] branch master updated: ARROW-16448: [CI][Archery] Refactor EmailReport to be a JinjaReport
This is an automated email from the ASF dual-hosted git repository.
kszucs pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new be08e92f52 ARROW-16448: [CI][Archery] Refactor EmailReport to be a JinjaReport
be08e92f52 is described below
commit be08e92f5216b451229eef649f495c049d03d64d
Author: Raúl Cumplido <ra...@gmail.com>
AuthorDate: Fri May 6 20:20:32 2022 +0200
ARROW-16448: [CI][Archery] Refactor EmailReport to be a JinjaReport
This PR tries to move the current `EmailReport` used for the Nightly report emails to a `JinjaReport` and adds a test for the report generation.
I also have tested locally sending an email successfully.
Closes #13074 from raulcd/ARROW-16448
Authored-by: Raúl Cumplido <ra...@gmail.com>
Signed-off-by: Krisztián Szűcs <sz...@gmail.com>
---
dev/archery/archery/crossbow/cli.py | 12 ++-
dev/archery/archery/crossbow/reports.py | 109 +++------------------
.../crossbow/tests/fixtures/email-report.txt | 27 +++++
dev/archery/archery/crossbow/tests/test_reports.py | 17 +++-
.../archery/templates/email_nightly_report.txt.j2 | 59 +++++++++++
5 files changed, 124 insertions(+), 100 deletions(-)
diff --git a/dev/archery/archery/crossbow/cli.py b/dev/archery/archery/crossbow/cli.py
index 6005c22de9..07d00ab3bd 100644
--- a/dev/archery/archery/crossbow/cli.py
+++ b/dev/archery/archery/crossbow/cli.py
@@ -279,8 +279,8 @@ def report(obj, job_name, sender_name, sender_email, recipient_email,
queue.fetch()
job = queue.get(job_name)
- report = EmailReport(
- job=job,
+ email_report = EmailReport(
+ report=Report(job),
sender_name=sender_name,
sender_email=sender_email,
recipient_email=recipient_email
@@ -293,14 +293,16 @@ def report(obj, job_name, sender_name, sender_email, recipient_email,
)
if send:
- report.send(
+ ReportUtils.send_email(
smtp_user=smtp_user,
smtp_password=smtp_password,
smtp_server=smtp_server,
- smtp_port=smtp_port
+ smtp_port=smtp_port,
+ recipient_email=recipient_email,
+ message=email_report.render("text")
)
else:
- report.show(output)
+ output.write(email_report.render("text"))
@crossbow.command()
diff --git a/dev/archery/archery/crossbow/reports.py b/dev/archery/archery/crossbow/reports.py
index 3858106a90..969c6c4f72 100644
--- a/dev/archery/archery/crossbow/reports.py
+++ b/dev/archery/archery/crossbow/reports.py
@@ -20,8 +20,6 @@ import collections
import operator
import fnmatch
import functools
-from io import StringIO
-import textwrap
import requests
@@ -194,105 +192,30 @@ class ReportUtils:
)
return resp
-
-class EmailReport(Report):
-
- HEADER = textwrap.dedent("""
- Arrow Build Report for Job {job_name}
-
- All tasks: {all_tasks_url}
- """)
-
- TASK = textwrap.dedent("""
- - {name}:
- URL: {url}
- """).strip()
-
- EMAIL = textwrap.dedent("""
- From: {sender_name} <{sender_email}>
- To: {recipient_email}
- Subject: {subject}
-
- {body}
- """).strip()
-
- STATUS_HEADERS = {
- # from CombinedStatus
- 'error': 'Errored Tasks:',
- 'failure': 'Failed Tasks:',
- 'pending': 'Pending Tasks:',
- 'success': 'Succeeded Tasks:',
- }
-
- def __init__(self, job, sender_name, sender_email, recipient_email):
- self.sender_name = sender_name
- self.sender_email = sender_email
- self.recipient_email = recipient_email
- super().__init__(job)
-
- def listing(self, tasks):
- return '\n'.join(
- sorted(
- self.TASK.format(
- name=task_name,
- url=self.task_url(task)
- )
- for task_name, task in tasks.items()
- )
- )
-
- def header(self):
- url = self.url(self.job.branch)
- return self.HEADER.format(job_name=self.job.branch, all_tasks_url=url)
-
- def subject(self):
- failures = len(self.tasks_by_state.get("failure", []))
- errors = len(self.tasks_by_state.get("error", []))
- pending = len(self.tasks_by_state.get("pending", []))
- return (
- f"[NIGHTLY] Arrow Build Report for Job {self.job.branch}: "
- f"{failures+errors} failed, {pending} pending"
- )
-
- def body(self):
- buffer = StringIO()
- buffer.write(self.header())
-
- for state in ('failure', 'error', 'pending', 'success'):
- if state in self.tasks_by_state:
- tasks = self.tasks_by_state[state]
- buffer.write('\n')
- buffer.write(self.STATUS_HEADERS[state])
- buffer.write('\n')
- buffer.write(self.listing(tasks))
- buffer.write('\n')
-
- return buffer.getvalue()
-
- def email(self):
- return self.EMAIL.format(
- sender_name=self.sender_name,
- sender_email=self.sender_email,
- recipient_email=self.recipient_email,
- subject=self.subject(),
- body=self.body()
- )
-
- def show(self, outstream):
- outstream.write(self.email())
-
- def send(self, smtp_user, smtp_password, smtp_server, smtp_port):
+ @classmethod
+ def send_email(cls, smtp_user, smtp_password, smtp_server, smtp_port,
+ recipient_email, message):
import smtplib
- email = self.email()
-
server = smtplib.SMTP_SSL(smtp_server, smtp_port)
server.ehlo()
server.login(smtp_user, smtp_password)
- server.sendmail(smtp_user, self.recipient_email, email)
+ server.sendmail(smtp_user, recipient_email, message)
server.close()
+class EmailReport(JinjaReport):
+ templates = {
+ 'text': 'email_nightly_report.txt.j2',
+ }
+ fields = [
+ 'report',
+ 'sender_name',
+ 'sender_email',
+ 'recipient_email',
+ ]
+
+
class CommentReport(Report):
_markdown_badge = '[]({url})'
diff --git a/dev/archery/archery/crossbow/tests/fixtures/email-report.txt b/dev/archery/archery/crossbow/tests/fixtures/email-report.txt
new file mode 100644
index 0000000000..9eee0a3246
--- /dev/null
+++ b/dev/archery/archery/crossbow/tests/fixtures/email-report.txt
@@ -0,0 +1,27 @@
+From: Sender Reporter <se...@arrow.com>
+To: recipient@arrow.com
+Subject: [NIGHTLY] Arrow Build Report for Job ursabot-1: 2 failed, 1 pending
+
+Arrow Build Report for Job ursabot-1
+
+All tasks: https://github.com/apache/crossbow/branches/all?query=ursabot-1
+
+Failed Tasks:
+
+- wheel-osx-cp37m
+ https://github.com/apache/crossbow/runs/2
+
+Errored Tasks:
+
+- wheel-osx-cp36m
+ https://github.com/apache/crossbow/runs/3
+
+Pending Tasks:
+
+- wheel-win-cp36m
+ https://github.com/apache/crossbow/runs/4
+
+Succeeded Tasks:
+
+- docker-cpp-cmake32
+ https://github.com/apache/crossbow/runs/1
diff --git a/dev/archery/archery/crossbow/tests/test_reports.py b/dev/archery/archery/crossbow/tests/test_reports.py
index deb90c47cf..91d98cfb9a 100644
--- a/dev/archery/archery/crossbow/tests/test_reports.py
+++ b/dev/archery/archery/crossbow/tests/test_reports.py
@@ -18,7 +18,8 @@
import textwrap
from archery.crossbow.core import yaml
-from archery.crossbow.reports import ChatReport, CommentReport, Report
+from archery.crossbow.reports import (ChatReport, CommentReport, EmailReport,
+ Report)
def test_crossbow_comment_formatter(load_fixture):
@@ -35,7 +36,7 @@ def test_crossbow_comment_formatter(load_fixture):
assert report.show() == textwrap.dedent(expected).strip()
-def test_crossbow_report(load_fixture):
+def test_crossbow_chat_report(load_fixture):
expected_msg = load_fixture('chat-report.txt')
job = load_fixture('crossbow-job.yaml', decoder=yaml.load)
report = Report(job)
@@ -43,3 +44,15 @@ def test_crossbow_report(load_fixture):
report_chat = ChatReport(report=report)
assert report_chat.render("text") == textwrap.dedent(expected_msg)
+
+
+def test_crossbow_email_report(load_fixture):
+ expected_msg = load_fixture('email-report.txt')
+ job = load_fixture('crossbow-job.yaml', decoder=yaml.load)
+ report = Report(job)
+ assert report.tasks_by_state is not None
+ empail_report = EmailReport(report=report, sender_name="Sender Reporter",
+ sender_email="sender@arrow.com",
+ recipient_email="recipient@arrow.com")
+
+ assert empail_report.render("text") == textwrap.dedent(expected_msg)
diff --git a/dev/archery/archery/templates/email_nightly_report.txt.j2 b/dev/archery/archery/templates/email_nightly_report.txt.j2
new file mode 100644
index 0000000000..7e068f202d
--- /dev/null
+++ b/dev/archery/archery/templates/email_nightly_report.txt.j2
@@ -0,0 +1,59 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#}
+{%- if True -%}
+{%- endif -%}
+From: {{ sender_name }} <{{ sender_email }}>
+To: {{ recipient_email }}
+Subject: [NIGHTLY] Arrow Build Report for Job {{report.job.branch}}: {{ (report.tasks_by_state["error"] | length) + (report.tasks_by_state["failure"] | length) }} failed, {{ report.tasks_by_state["pending"] | length }} pending
+
+Arrow Build Report for Job {{ report.job.branch }}
+
+All tasks: {{ report.url(report.job.branch) }}
+{% if report.tasks_by_state["failure"] %}
+Failed Tasks:
+
+{% for task_name, task in report.tasks_by_state["failure"] | dictsort -%}
+- {{ task_name }}
+ {{ report.task_url(task) }}
+{% endfor %}
+{% endif %}
+{%- if report.tasks_by_state["error"] -%}
+Errored Tasks:
+
+{% for task_name, task in report.tasks_by_state["error"] | dictsort -%}
+- {{ task_name }}
+ {{ report.task_url(task) }}
+{% endfor %}
+{% endif %}
+{%- if report.tasks_by_state["pending"] -%}
+Pending Tasks:
+
+{% for task_name, task in report.tasks_by_state["pending"] | dictsort -%}
+- {{ task_name }}
+ {{ report.task_url(task) }}
+{% endfor %}
+{% endif %}
+{%- if report.tasks_by_state["success"] -%}
+Succeeded Tasks:
+
+{% for task_name, task in report.tasks_by_state["success"] | dictsort -%}
+- {{ task_name }}
+ {{ report.task_url(task) }}
+{% endfor %}
+{%- endif -%}
\ No newline at end of file