You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2024/07/04 15:39:41 UTC

(airflow) branch main updated: Enhancing changelog generation for provider releases (#40552)

This is an automated email from the ASF dual-hosted git repository.

potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 9918f2a182 Enhancing changelog generation for provider releases (#40552)
9918f2a182 is described below

commit 9918f2a182d4fb735e964660ffeab84bccb73b8c
Author: Amogh Desai <am...@gmail.com>
AuthorDate: Thu Jul 4 21:09:33 2024 +0530

    Enhancing changelog generation for provider releases (#40552)
---
 .../prepare_providers/provider_documentation.py    | 112 +++++++++++++++------
 dev/breeze/tests/test_provider_documentation.py    |  48 +++++++--
 2 files changed, 124 insertions(+), 36 deletions(-)

diff --git a/dev/breeze/src/airflow_breeze/prepare_providers/provider_documentation.py b/dev/breeze/src/airflow_breeze/prepare_providers/provider_documentation.py
index 4d0e8b8c0c..31e80d0ea7 100644
--- a/dev/breeze/src/airflow_breeze/prepare_providers/provider_documentation.py
+++ b/dev/breeze/src/airflow_breeze/prepare_providers/provider_documentation.py
@@ -98,6 +98,27 @@ Changelog
 Initial version of the provider.
 """
 
+SHORT_HASH_TO_TYPE_DICT = {}
+
+
+class TypeOfChange(Enum):
+    DOCUMENTATION = "d"
+    BUGFIX = "b"
+    FEATURE = "f"
+    BREAKING_CHANGE = "x"
+    SKIP = "s"
+
+
+# defines the precedence order for provider version bumps
+# BREAKING_CHANGE > FEATURE > BUGFIX > DOCUMENTATION > SKIP
+precedence_order = {
+    TypeOfChange.SKIP: 0,
+    TypeOfChange.DOCUMENTATION: 1,
+    TypeOfChange.BUGFIX: 2,
+    TypeOfChange.FEATURE: 3,
+    TypeOfChange.BREAKING_CHANGE: 4,
+}
+
 
 class Change(NamedTuple):
     """Stores details about commits"""
@@ -111,12 +132,16 @@ class Change(NamedTuple):
     pr: str | None
 
 
-class TypeOfChange(Enum):
-    DOCUMENTATION = "d"
-    BUGFIX = "b"
-    FEATURE = "f"
-    BREAKING_CHANGE = "x"
-    SKIP = "s"
+def get_most_impactful_change(changes):
+    changes_types = list(changes.values())
+    changes_enum = [TypeOfChange(change) for change in changes_types]
+    return max(changes_enum, key=lambda change: precedence_order[change])
+
+
+def format_message_for_classification(message):
+    num = re.search(r"#(\d+)", message).group(1)
+    new_message = re.sub(r"#(\d+)", f"https://github.com/apache/airflow/pull/{num}", message)
+    return new_message
 
 
 class ClassifiedChanges:
@@ -403,7 +428,7 @@ def _ask_the_user_for_the_type_of_changes(non_interactive: bool) -> TypeOfChange
     display_answers = "/".join(type_of_changes_array) + "/q"
     while True:
         get_console().print(
-            "[warning]Type of change (d)ocumentation, (b)ugfix, (f)eature, (x)breaking "
+            "[warning]Type of change (b)ugfix, (f)eature, (x)breaking "
             f"change, (s)kip, (q)uit [{display_answers}]?[/] ",
             end="",
         )
@@ -702,7 +727,36 @@ def update_release_notes(
             )
             raise PrepareReleaseDocsNoChangesException()
         else:
-            type_of_change = _ask_the_user_for_the_type_of_changes(non_interactive=non_interactive)
+            answer = user_confirm(
+                f"Does the provider: {provider_package_id} have any changes apart from 'doc-only'?"
+            )
+            if answer == Answer.NO:
+                _mark_latest_changes_as_documentation_only(provider_package_id, list_of_list_of_changes)
+                return with_breaking_changes, maybe_with_new_features
+            change_table_len = len(list_of_list_of_changes[0])
+            table_iter = 0
+            global SHORT_HASH_TO_TYPE_DICT
+            while table_iter < change_table_len:
+                get_console().print()
+                formatted_message = format_message_for_classification(
+                    list_of_list_of_changes[0][table_iter].message_without_backticks
+                )
+                get_console().print(
+                    f"[green]Define the type of change for "
+                    f"`{formatted_message}`"
+                    f" by referring to the above table[/]"
+                )
+                type_of_change = _ask_the_user_for_the_type_of_changes(non_interactive=non_interactive)
+                # update the type of change for every short_hash in the global dict
+                SHORT_HASH_TO_TYPE_DICT[list_of_list_of_changes[0][table_iter].short_hash] = type_of_change
+                table_iter += 1
+                print()
+
+            most_impactful = get_most_impactful_change(SHORT_HASH_TO_TYPE_DICT)
+            get_console().print(
+                f"[info]The version will be bumped because of {most_impactful} kind of change"
+            )
+            type_of_change = most_impactful
             if type_of_change == TypeOfChange.SKIP:
                 raise PrepareReleaseDocsUserSkippedException()
             get_console().print(
@@ -710,9 +764,7 @@ def update_release_notes(
                 f"[special]{TYPE_OF_CHANGE_DESCRIPTION[type_of_change]}"
             )
             get_console().print()
-            if type_of_change == TypeOfChange.DOCUMENTATION:
-                _mark_latest_changes_as_documentation_only(provider_package_id, list_of_list_of_changes)
-            elif type_of_change in [TypeOfChange.BUGFIX, TypeOfChange.FEATURE, TypeOfChange.BREAKING_CHANGE]:
+            if type_of_change in [TypeOfChange.BUGFIX, TypeOfChange.FEATURE, TypeOfChange.BREAKING_CHANGE]:
                 with_breaking_changes, maybe_with_new_features = _update_version_in_provider_yaml(
                     provider_package_id=provider_package_id, type_of_change=type_of_change
                 )
@@ -735,6 +787,7 @@ def update_release_notes(
         )
     else:
         answer = Answer.YES
+
     if answer == Answer.NO:
         type_of_change = _ask_the_user_for_the_type_of_changes(non_interactive=False)
         if type_of_change == TypeOfChange.SKIP:
@@ -812,33 +865,34 @@ def _find_insertion_index_for_version(content: list[str], version: str) -> tuple
 def _get_changes_classified(
     changes: list[Change], with_breaking_changes: bool, maybe_with_new_features: bool
 ) -> ClassifiedChanges:
-    """Pre-classifies changes based on commit message, it's wildly guessing now,
-
-    The classification also includes the decision made by the release manager when classifying the release.
+    """
+    Pre-classifies changes based on their type_of_change attribute derived based on release manager's call.
 
-    However, if we switch to semantic commits, it could be automated. This list
-    is supposed to be manually reviewed and re-classified by release manager
-    anyway.
+    The classification is based on the decision made by the release manager when classifying the release.
+    If we switch to semantic commits, this process could be automated. This list is still supposed to be
+    manually reviewed and re-classified by the release manager if needed.
 
-    :param changes: list of changes
-    :return: list of changes classified semi-automatically to the fix/feature/breaking/other buckets
+    :param changes: list of changes to be classified
+    :param with_breaking_changes: whether to include breaking changes in the classification
+    :param maybe_with_new_features: whether to include new features in the classification
+    :return: ClassifiedChanges object containing changes classified into fixes, features, breaking changes,
+    misc.
     """
     classified_changes = ClassifiedChanges()
     for change in changes:
-        # Special cases
-        if "bump minimum Airflow version in providers" in change.message.lower():
-            classified_changes.misc.append(change)
-        # General cases
-        elif "fix" in change.message.lower():
+        type_of_change = None
+        if change.short_hash in SHORT_HASH_TO_TYPE_DICT:
+            type_of_change = SHORT_HASH_TO_TYPE_DICT[change.short_hash]
+
+        if type_of_change == TypeOfChange.BUGFIX:
             classified_changes.fixes.append(change)
-        elif "misc" in change.message.lower():
-            classified_changes.misc.append(change)
-        elif "add" in change.message.lower() and maybe_with_new_features:
+        elif type_of_change == TypeOfChange.FEATURE and maybe_with_new_features:
             classified_changes.features.append(change)
-        elif "breaking" in change.message.lower() and with_breaking_changes:
+        elif type_of_change == TypeOfChange.BREAKING_CHANGE and with_breaking_changes:
             classified_changes.breaking_changes.append(change)
         else:
-            classified_changes.other.append(change)
+            # for the changes we do not have a category for, add it to misc
+            classified_changes.misc.append(change)
     return classified_changes
 
 
diff --git a/dev/breeze/tests/test_provider_documentation.py b/dev/breeze/tests/test_provider_documentation.py
index c3508cca2d..19fbdbb9b1 100644
--- a/dev/breeze/tests/test_provider_documentation.py
+++ b/dev/breeze/tests/test_provider_documentation.py
@@ -16,10 +16,14 @@
 # under the License.
 from __future__ import annotations
 
+import random
+import string
+
 import pytest
 
 from airflow_breeze.prepare_providers.provider_documentation import (
     Change,
+    TypeOfChange,
     _convert_git_changes_to_table,
     _find_insertion_index_for_version,
     _get_change_from_line,
@@ -217,15 +221,36 @@ def test_verify_changelog_exists():
     )
 
 
+def generate_random_string(length):
+    c = string.hexdigits.lower()
+    return "".join(random.choice(c) for _ in range(length))
+
+
+def generate_long_hash():
+    return generate_random_string(40)
+
+
+def generate_short_hash():
+    return generate_random_string(10)
+
+
 @pytest.mark.parametrize(
     "descriptions, with_breaking_changes, maybe_with_new_features,"
-    "breaking_count, feature_count, bugfix_count, other_count",
+    "breaking_count, feature_count, bugfix_count, other_count, type_of_change",
     [
-        (["Added feature x"], True, True, 0, 1, 0, 0),
-        (["Added feature x"], False, False, 0, 0, 0, 1),
-        (["Breaking change in"], True, True, 1, 0, 0, 0),
-        (["Breaking change in", "Added feature y"], True, True, 1, 1, 0, 0),
-        (["Fix change in", "Breaking feature y"], False, True, 0, 0, 1, 1),
+        (["Added feature x"], True, True, 0, 1, 0, 0, [TypeOfChange.FEATURE]),
+        (["Added feature x"], False, True, 0, 1, 0, 0, [TypeOfChange.FEATURE]),
+        (["Breaking change in"], True, True, 1, 0, 0, 0, [TypeOfChange.BREAKING_CHANGE]),
+        (
+            ["Fix change in", "Breaking feature y"],
+            True,
+            False,
+            1,
+            0,
+            1,
+            0,
+            [TypeOfChange.BUGFIX, TypeOfChange.BREAKING_CHANGE],
+        ),
     ],
 )
 def test_classify_changes_automatically(
@@ -236,12 +261,21 @@ def test_classify_changes_automatically(
     feature_count: int,
     bugfix_count: int,
     other_count: int,
+    type_of_change: TypeOfChange,
 ):
+    from airflow_breeze.prepare_providers.provider_documentation import SHORT_HASH_TO_TYPE_DICT
+
     """Test simple automated classification of the changes based on their single-line description."""
     changes = [
-        _get_change_from_line(f"LONG SHORT 2023-12-01 {description}", version="0.1.0")
+        _get_change_from_line(
+            f"{generate_long_hash()} {generate_short_hash()} 2023-12-01 {description}", version="0.1.0"
+        )
         for description in descriptions
     ]
+
+    for i in range(0, len(changes)):
+        SHORT_HASH_TO_TYPE_DICT[changes[i].short_hash] = type_of_change[i]
+
     classified_changes = _get_changes_classified(
         changes, with_breaking_changes=with_breaking_changes, maybe_with_new_features=maybe_with_new_features
     )