You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@celeborn.apache.org by zh...@apache.org on 2023/06/16 10:13:14 UTC
[incubator-celeborn] branch branch-0.3 updated (5d0cc36bb -> d9c3fb240)
This is an automated email from the ASF dual-hosted git repository.
zhouky pushed a change to branch branch-0.3
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
from 5d0cc36bb [CELEBORN-644][HELM] Support Helm Deploy Celeborn with HostNetwork And DnsPolicy
new 24222f2a2 [CELEBORN-105][TEST] Kubernetes Integration Test
new d9c3fb240 [CELEBORN-633][INFRA] Introduce PR merge script
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/integration.yml | 78 +++
dev/merge_pr.py | 578 +++++++++++++++++++++
pom.xml | 14 +
requirements.txt | 6 +
tests/{flink-it => kubernetes-it}/pom.xml | 43 +-
.../src/test/resources/log4j2-test.xml | 4 -
.../DeploySuite.scala | 59 +++
.../WithMiniKube.scala | 21 +-
8 files changed, 749 insertions(+), 54 deletions(-)
create mode 100644 .github/workflows/integration.yml
create mode 100755 dev/merge_pr.py
copy tests/{flink-it => kubernetes-it}/pom.xml (65%)
copy {client => tests/kubernetes-it}/src/test/resources/log4j2-test.xml (89%)
create mode 100644 tests/kubernetes-it/src/test/scala/org.apache.celeborn.integration/DeploySuite.scala
copy client/src/main/java/org/apache/celeborn/client/read/PartitionReader.java => tests/kubernetes-it/src/test/scala/org.apache.celeborn.integration/WithMiniKube.scala (70%)
[incubator-celeborn] 02/02: [CELEBORN-633][INFRA] Introduce PR merge script
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhouky pushed a commit to branch branch-0.3
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
commit d9c3fb240c6af6125d35cf9e9780f20c83d96549
Author: Cheng Pan <ch...@apache.org>
AuthorDate: Fri Jun 2 19:52:04 2023 +0800
[CELEBORN-633][INFRA] Introduce PR merge script
### What changes were proposed in this pull request?
Introduce PR merge script `dev/merge_pr.py`, which is borrowed from Apache Spark
### Why are the changes needed?
This script simplifies the PR merge procedure
- auto backport to release branches
- auto close the JIRA ticket
- auto fill in the JIRA fixed version
- reserve the PR description in git log
- reserve the author and committer in git log
### Does this PR introduce _any_ user-facing change?
No, it's for committers.
### How was this patch tested?
https://github.com/apache/incubator-celeborn/commit/a1de16a80f7d9a5e854a978eb48228b292c3ca41 was merged by this tool
Closes #1539 from pan3793/CELEBORN-633.
Authored-by: Cheng Pan <ch...@apache.org>
Signed-off-by: Cheng Pan <ch...@apache.org>
---
dev/merge_pr.py | 578 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
requirements.txt | 6 +
2 files changed, 584 insertions(+)
diff --git a/dev/merge_pr.py b/dev/merge_pr.py
new file mode 100755
index 000000000..e5b09e136
--- /dev/null
+++ b/dev/merge_pr.py
@@ -0,0 +1,578 @@
+#!/usr/bin/env python3
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Utility for creating well-formed pull request merges and pushing them to Apache
+# Celeborn.
+# usage: dev/merge_pr.py (see config env vars below)
+#
+# This utility assumes you already have a local Celeborn git folder and that you
+# have added remotes corresponding to both (i) the github apache Celeborn
+# mirror and (ii) the apache git repo.
+
+import json
+import os
+import re
+import subprocess
+import sys
+import traceback
+from urllib.request import urlopen
+from urllib.request import Request
+from urllib.error import HTTPError
+
+try:
+ import jira.client
+
+ JIRA_IMPORTED = True
+except ImportError:
+ JIRA_IMPORTED = False
+
+# Location of your Celeborn git development area
+PROJECT_HOME = os.environ.get("PROJECT_HOME", os.getcwd())
+# Remote name which points to the GitHub site
+PR_REMOTE_NAME = os.environ.get("PR_REMOTE_NAME", "apache")
+# Remote name which points to Apache git
+PUSH_REMOTE_NAME = os.environ.get("PUSH_REMOTE_NAME", "apache")
+# ASF JIRA username
+ASF_USERNAME = os.environ.get("ASF_USERNAME", "")
+# ASF JIRA password
+ASF_PASSWORD = os.environ.get("ASF_PASSWORD", "")
+# OAuth key used for issuing requests against the GitHub API. If this is not defined, then requests
+# will be unauthenticated. You should only need to configure this if you find yourself regularly
+# exceeding your IP's unauthenticated request rate limit. You can create an OAuth key at
+# https://github.com/settings/tokens. This script only requires the "public_repo" scope.
+GITHUB_OAUTH_KEY = os.environ.get("GITHUB_OAUTH_KEY")
+
+
+GITHUB_BASE = "https://github.com/apache/incubator-celeborn/pull"
+GITHUB_API_BASE = "https://api.github.com/repos/apache/incubator-celeborn"
+JIRA_BASE = "https://issues.apache.org/jira/browse"
+JIRA_API_BASE = "https://issues.apache.org/jira"
+# Prefix added to temporary branches
+BRANCH_PREFIX = "PR_TOOL"
+
+
+def get_json(url):
+ try:
+ request = Request(url)
+ if GITHUB_OAUTH_KEY:
+ request.add_header("Authorization", "token %s" % GITHUB_OAUTH_KEY)
+ return json.load(urlopen(request))
+ except HTTPError as e:
+ if "X-RateLimit-Remaining" in e.headers and e.headers["X-RateLimit-Remaining"] == "0":
+ print(
+ "Exceeded the GitHub API rate limit; see the instructions in "
+ + "dev/merge_pr.py to configure an OAuth token for making authenticated "
+ + "GitHub requests."
+ )
+ else:
+ print("Unable to fetch URL, exiting: %s" % url)
+ sys.exit(-1)
+
+
+def fail(msg):
+ print(msg)
+ clean_up()
+ sys.exit(-1)
+
+
+def run_cmd(cmd):
+ print(cmd)
+ if isinstance(cmd, list):
+ return subprocess.check_output(cmd).decode("utf-8")
+ else:
+ return subprocess.check_output(cmd.split(" ")).decode("utf-8")
+
+
+def continue_maybe(prompt):
+ result = input("\n%s (y/n): " % prompt)
+ if result.lower() != "y":
+ fail("Okay, exiting")
+
+
+def clean_up():
+ if "original_head" in globals():
+ print("Restoring head pointer to %s" % original_head)
+ run_cmd("git checkout %s" % original_head)
+
+ branches = run_cmd("git branch").replace(" ", "").split("\n")
+
+ for branch in list(filter(lambda x: x.startswith(BRANCH_PREFIX), branches)):
+ print("Deleting local branch %s" % branch)
+ run_cmd("git branch -D %s" % branch)
+
+
+# merge the requested PR and return the merge hash
+def merge_pr(pr_num, target_ref, title, body, pr_repo_desc):
+ pr_branch_name = "%s_MERGE_PR_%s" % (BRANCH_PREFIX, pr_num)
+ target_branch_name = "%s_MERGE_PR_%s_%s" % (BRANCH_PREFIX, pr_num, target_ref.upper())
+ run_cmd("git fetch %s pull/%s/head:%s" % (PR_REMOTE_NAME, pr_num, pr_branch_name))
+ run_cmd("git fetch %s %s:%s" % (PUSH_REMOTE_NAME, target_ref, target_branch_name))
+ run_cmd("git checkout %s" % target_branch_name)
+
+ had_conflicts = False
+ try:
+ run_cmd(["git", "merge", pr_branch_name, "--squash"])
+ except Exception as e:
+ msg = "Error merging: %s\nWould you like to manually fix-up this merge?" % e
+ continue_maybe(msg)
+ msg = "Okay, please fix any conflicts and 'git add' conflicting files... Finished?"
+ continue_maybe(msg)
+ had_conflicts = True
+
+ # First commit author should be considered as the primary author when the rank is the same
+ commit_authors = run_cmd(
+ ["git", "log", "HEAD..%s" % pr_branch_name, "--pretty=format:%an <%ae>", "--reverse"]
+ ).split("\n")
+ distinct_authors = sorted(
+ list(dict.fromkeys(commit_authors)), key=lambda x: commit_authors.count(x), reverse=True
+ )
+ primary_author = input(
+ 'Enter primary author in the format of "name <email>" [%s]: ' % distinct_authors[0]
+ )
+ if primary_author == "":
+ primary_author = distinct_authors[0]
+ else:
+ # When primary author is specified manually, de-dup it from author list and
+ # put it at the head of author list.
+ distinct_authors = list(filter(lambda x: x != primary_author, distinct_authors))
+ distinct_authors.insert(0, primary_author)
+
+ merge_message_flags = []
+
+ merge_message_flags += ["-m", title]
+ if body is not None:
+ # We remove @ symbols from the body to avoid triggering e-mails
+ # to people every time someone creates a public fork of Celeborn.
+ merge_message_flags += ["-m", body.replace("@", "")]
+
+ committer_name = run_cmd("git config --get user.name").strip()
+ committer_email = run_cmd("git config --get user.email").strip()
+
+ if had_conflicts:
+ message = "This patch had conflicts when merged, resolved by\nCommitter: %s <%s>" % (
+ committer_name,
+ committer_email,
+ )
+ merge_message_flags += ["-m", message]
+
+ # The string "Closes #%s" string is required for GitHub to correctly close the PR
+ merge_message_flags += ["-m", "Closes #%s from %s." % (pr_num, pr_repo_desc)]
+
+ authors = "Authored-by:" if len(distinct_authors) == 1 else "Lead-authored-by:"
+ authors += " %s" % (distinct_authors.pop(0))
+ if len(distinct_authors) > 0:
+ authors += "\n" + "\n".join(["Co-authored-by: %s" % a for a in distinct_authors])
+ authors += "\n" + "Signed-off-by: %s <%s>" % (committer_name, committer_email)
+
+ merge_message_flags += ["-m", authors]
+
+ run_cmd(["git", "commit", '--author="%s"' % primary_author] + merge_message_flags)
+
+ continue_maybe(
+ "Merge complete (local ref %s). Push to %s?" % (target_branch_name, PUSH_REMOTE_NAME)
+ )
+
+ try:
+ run_cmd("git push %s %s:%s" % (PUSH_REMOTE_NAME, target_branch_name, target_ref))
+ except Exception as e:
+ clean_up()
+ fail("Exception while pushing: %s" % e)
+
+ merge_hash = run_cmd("git rev-parse %s" % target_branch_name)[:8]
+ clean_up()
+ print("Pull request #%s merged!" % pr_num)
+ print("Merge hash: %s" % merge_hash)
+ return merge_hash
+
+
+def cherry_pick(pr_num, merge_hash, default_branch):
+ pick_ref = input("Enter a branch name [%s]: " % default_branch)
+ if pick_ref == "":
+ pick_ref = default_branch
+
+ pick_branch_name = "%s_PICK_PR_%s_%s" % (BRANCH_PREFIX, pr_num, pick_ref.upper())
+
+ run_cmd("git fetch %s %s:%s" % (PUSH_REMOTE_NAME, pick_ref, pick_branch_name))
+ run_cmd("git checkout %s" % pick_branch_name)
+
+ try:
+ run_cmd("git cherry-pick -sx %s" % merge_hash)
+ except Exception as e:
+ msg = "Error cherry-picking: %s\nWould you like to manually fix-up this merge?" % e
+ continue_maybe(msg)
+ msg = "Okay, please fix any conflicts and finish the cherry-pick. Finished?"
+ continue_maybe(msg)
+
+ continue_maybe(
+ "Pick complete (local ref %s). Push to %s?" % (pick_branch_name, PUSH_REMOTE_NAME)
+ )
+
+ try:
+ run_cmd("git push %s %s:%s" % (PUSH_REMOTE_NAME, pick_branch_name, pick_ref))
+ except Exception as e:
+ clean_up()
+ fail("Exception while pushing: %s" % e)
+
+ pick_hash = run_cmd("git rev-parse %s" % pick_branch_name)[:8]
+ clean_up()
+
+ print("Pull request #%s picked into %s!" % (pr_num, pick_ref))
+ print("Pick hash: %s" % pick_hash)
+ return pick_ref
+
+
+def fix_version_from_branch(branch, versions):
+ # Note: Assumes this is a sorted (newest->oldest) list of un-released versions
+ if branch == "main":
+ return versions[0]
+ else:
+ branch_ver = branch.replace("branch-", "")
+ return list(filter(lambda x: x.name.startswith(branch_ver), versions))[-1]
+
+
+def resolve_jira_issue(merge_branches, comment, default_jira_id=""):
+ asf_jira = jira.client.JIRA(
+ {"server": JIRA_API_BASE}, basic_auth=(ASF_USERNAME, ASF_PASSWORD)
+ )
+
+ jira_id = input("Enter a JIRA id [%s]: " % default_jira_id)
+ if jira_id == "":
+ jira_id = default_jira_id
+
+ try:
+ issue = asf_jira.issue(jira_id)
+ except Exception as e:
+ fail("ASF JIRA could not find %s\n%s" % (jira_id, e))
+
+ cur_status = issue.fields.status.name
+ cur_summary = issue.fields.summary
+ cur_assignee = issue.fields.assignee
+ if cur_assignee is None:
+ cur_assignee = choose_jira_assignee(issue, asf_jira)
+ # Check again, we might not have chosen an assignee
+ if cur_assignee is None:
+ cur_assignee = "NOT ASSIGNED!!!"
+ else:
+ cur_assignee = cur_assignee.displayName
+
+ if cur_status == "Resolved" or cur_status == "Closed":
+ fail("JIRA issue %s already has status '%s'" % (jira_id, cur_status))
+ print("=== JIRA %s ===" % jira_id)
+ print(
+ "summary\t\t%s\nassignee\t%s\nstatus\t\t%s\nurl\t\t%s/%s\n"
+ % (cur_summary, cur_assignee, cur_status, JIRA_BASE, jira_id)
+ )
+
+ versions = asf_jira.project_versions("CELEBORN")
+ versions = sorted(versions, key=lambda x: x.name, reverse=True)
+ versions = list(filter(lambda x: x.raw["released"] is False, versions))
+ # Consider only x.y.z versions
+ versions = list(filter(lambda x: re.match(r"\d+\.\d+\.\d+", x.name), versions))
+
+ default_fix_versions = list(
+ map(lambda x: fix_version_from_branch(x, versions).name, merge_branches)
+ )
+ for v in default_fix_versions:
+ # Handles the case where we have forked a release branch but not yet made the release.
+ # In this case, if the PR is committed to the main branch and the release branch, we
+ # only consider the release branch to be the fix version. E.g. it is not valid to have
+ # both 1.1.0 and 1.0.0 as fix versions.
+ (major, minor, patch) = v.split(".")
+ if patch == "0":
+ previous = "%s.%s.%s" % (major, int(minor) - 1, 0)
+ if previous in default_fix_versions:
+ default_fix_versions = list(filter(lambda x: x != v, default_fix_versions))
+ default_fix_versions = ",".join(default_fix_versions)
+
+ available_versions = set(list(map(lambda v: v.name, versions)))
+ while True:
+ try:
+ fix_versions = input(
+ "Enter comma-separated fix version(s) [%s]: " % default_fix_versions
+ )
+ if fix_versions == "":
+ fix_versions = default_fix_versions
+ fix_versions = fix_versions.replace(" ", "").split(",")
+ if set(fix_versions).issubset(available_versions):
+ break
+ else:
+ print(
+ "Specified version(s) [%s] not found in the available versions, try "
+ "again (or leave blank and fix manually)." % (", ".join(fix_versions))
+ )
+ except KeyboardInterrupt:
+ raise
+ except BaseException:
+ traceback.print_exc()
+ print("Error setting fix version(s), try again (or leave blank and fix manually)")
+
+ def get_version_json(version_str):
+ return list(filter(lambda v: v.name == version_str, versions))[0].raw
+
+ jira_fix_versions = list(map(lambda v: get_version_json(v), fix_versions))
+
+ resolve = list(filter(lambda a: a["name"] == "Resolve Issue", asf_jira.transitions(jira_id)))[0]
+ resolution = list(filter(lambda r: r.raw["name"] == "Fixed", asf_jira.resolutions()))[0]
+ asf_jira.transition_issue(
+ jira_id,
+ resolve["id"],
+ fixVersions=jira_fix_versions,
+ comment=comment,
+ resolution={"id": resolution.raw["id"]},
+ )
+
+ print("Successfully resolved %s with fixVersions=%s!" % (jira_id, fix_versions))
+
+
+def choose_jira_assignee(issue, asf_jira):
+ """
+ Prompt the user to choose who to assign the issue to in jira, given a list of candidates,
+ including the original reporter and all commentors
+ """
+ while True:
+ try:
+ reporter = issue.fields.reporter
+ commentors = list(map(lambda x: x.author, issue.fields.comment.comments))
+ candidates = set(commentors)
+ candidates.add(reporter)
+ candidates = list(candidates)
+ print("JIRA is unassigned, choose assignee")
+ for idx, author in enumerate(candidates):
+ if "celeborn" in author.key:
+ continue
+ annotations = ["Reporter"] if author == reporter else []
+ if author in commentors:
+ annotations.append("Commentor")
+ print("[%d] %s (%s)" % (idx, author.displayName, ",".join(annotations)))
+ raw_assignee = input(
+ "Enter number of user, or userid, to assign to (blank to leave unassigned):"
+ )
+ if raw_assignee == "":
+ return None
+ else:
+ try:
+ id = int(raw_assignee)
+ assignee = candidates[id]
+ except BaseException:
+ # assume it's a user id, and try to assign (might fail, we just prompt again)
+ assignee = asf_jira.user(raw_assignee)
+ asf_jira.assign_issue(issue.key, assignee.name)
+ return assignee
+ except KeyboardInterrupt:
+ raise
+ except BaseException:
+ traceback.print_exc()
+ print("Error assigning JIRA, try again (or leave blank and fix manually)")
+
+
+def resolve_jira_issues(title, merge_branches, comment):
+ jira_ids = re.findall("CELEBORN-[0-9]{3,6}", title)
+
+ if len(jira_ids) == 0:
+ resolve_jira_issue(merge_branches, comment)
+ for jira_id in jira_ids:
+ resolve_jira_issue(merge_branches, comment, jira_id)
+
+
+def standardize_jira_ref(text):
+ """
+ Standardize the [CELEBORN-XXX][MODULE] prefix
+ Converts "[CELEBORN-XXX][core] Issue", "[Core] CELEBORN-XXX. Issue" or "CELEBORN XXX [CORE]: Issue" to
+ "[CELEBORN-XXX][CORE] Issue"
+ """
+ jira_refs = []
+ components = []
+
+ # If the string is compliant, no need to process any further
+ if re.search(r"^\[CELEBORN-[0-9]{3,6}\](\[[A-Z0-9_\s,]+\] )+\S+", text):
+ return text
+
+ # Extract JIRA ref(s):
+ pattern = re.compile(r"(CELEBORN[-\s]*[0-9]{3,6})+", re.IGNORECASE)
+ for ref in pattern.findall(text):
+ # Add brackets, replace spaces with a dash, & convert to uppercase
+ jira_refs.append("[" + re.sub(r"\s+", "-", ref.upper()) + "]")
+ text = text.replace(ref, "")
+
+ # Extract celeborn component(s):
+ # Look for alphanumeric chars, spaces, dashes, periods, and/or commas
+ pattern = re.compile(r"(\[[\w\s,.-]+\])", re.IGNORECASE)
+ for component in pattern.findall(text):
+ components.append(component.upper())
+ text = text.replace(component, "")
+
+ # Cleanup any remaining symbols:
+ pattern = re.compile(r"^\W+(.*)", re.IGNORECASE)
+ if pattern.search(text) is not None:
+ text = pattern.search(text).groups()[0]
+
+ # Assemble full text (JIRA ref(s), module(s), remaining text)
+ clean_text = "".join(jira_refs).strip() + "".join(components).strip() + " " + text.strip()
+
+ # Replace multiple spaces with a single space, e.g. if no jira refs and/or components were
+ # included
+ clean_text = re.sub(r"\s+", " ", clean_text.strip())
+
+ return clean_text
+
+
+def get_current_ref():
+ ref = run_cmd("git rev-parse --abbrev-ref HEAD").strip()
+ if ref == "HEAD":
+ # The current ref is a detached HEAD, so grab its SHA.
+ return run_cmd("git rev-parse HEAD").strip()
+ else:
+ return ref
+
+
+def main():
+ global original_head
+
+ os.chdir(PROJECT_HOME)
+ original_head = get_current_ref()
+
+ # Check this up front to avoid failing the JIRA update at the very end
+ if not ASF_USERNAME or not ASF_PASSWORD:
+ continue_maybe("The env-vars ASF_USERNAME and/or ASF_PASSWORD are not set. Continue?")
+
+ branches = get_json("%s/branches" % GITHUB_API_BASE)
+ branch_names = list(filter(lambda x: x.startswith("branch-"), [x["name"] for x in branches]))
+ # Assumes branch names can be sorted lexicographically
+ latest_branch = sorted(branch_names, reverse=True)[0]
+
+ pr_num = input("Which pull request would you like to merge? (e.g. 34): ")
+ pr = get_json("%s/pulls/%s" % (GITHUB_API_BASE, pr_num))
+ pr_events = get_json("%s/issues/%s/events" % (GITHUB_API_BASE, pr_num))
+
+ url = pr["url"]
+
+ # Warn if the PR is WIP
+ if "[WIP]" in pr["title"]:
+ msg = "The PR title has `[WIP]`:\n%s\nContinue?" % pr["title"]
+ continue_maybe(msg)
+
+ # Decide whether to use the modified title or not
+ modified_title = standardize_jira_ref(pr["title"]).rstrip(".")
+ if modified_title != pr["title"]:
+ print("I've re-written the title as follows to match the standard format:")
+ print("Original: %s" % pr["title"])
+ print("Modified: %s" % modified_title)
+ result = input("Would you like to use the modified title? (y/n): ")
+ if result.lower() == "y":
+ title = modified_title
+ print("Using modified title:")
+ else:
+ title = pr["title"]
+ print("Using original title:")
+ print(title)
+ else:
+ title = pr["title"]
+
+ body = pr["body"]
+ if body is None:
+ body = ""
+ modified_body = re.sub(re.compile(r"<!--[^>]*-->\n?", re.DOTALL), "", body).lstrip()
+ if modified_body != body:
+ print("=" * 80)
+ print(modified_body)
+ print("=" * 80)
+ print("I've removed the comments from PR template like the above:")
+ result = input("Would you like to use the modified body? (y/n): ")
+ if result.lower() == "y":
+ body = modified_body
+ print("Using modified body:")
+ else:
+ print("Using original body:")
+ print("=" * 80)
+ print(body)
+ print("=" * 80)
+ target_ref = pr["base"]["ref"]
+ user_login = pr["user"]["login"]
+ base_ref = pr["head"]["ref"]
+ pr_repo_desc = "%s/%s" % (user_login, base_ref)
+
+ # Merged pull requests don't appear as merged in the GitHub API;
+ # Instead, they're closed by asfgit.
+ merge_commits = [
+ e for e in pr_events if e["actor"]["login"] == "asfgit" and e["event"] == "closed"
+ ]
+
+ if merge_commits:
+ merge_hash = merge_commits[0]["commit_id"]
+ message = get_json("%s/commits/%s" % (GITHUB_API_BASE, merge_hash))["commit"]["message"]
+
+ print("Pull request %s has already been merged, assuming you want to backport" % pr_num)
+ commit_is_downloaded = (
+ run_cmd(["git", "rev-parse", "--quiet", "--verify", "%s^{commit}" % merge_hash]).strip()
+ != ""
+ )
+ if not commit_is_downloaded:
+ fail("Couldn't find any merge commit for #%s, you may need to update HEAD." % pr_num)
+
+ print("Found commit %s:\n%s" % (merge_hash, message))
+ cherry_pick(pr_num, merge_hash, latest_branch)
+ sys.exit(0)
+
+ if not bool(pr["mergeable"]):
+ msg = (
+ "Pull request %s is not mergeable in its current form.\n" % pr_num
+ + "Continue? (experts only!)"
+ )
+ continue_maybe(msg)
+
+ print("\n=== Pull Request #%s ===" % pr_num)
+ print("title\t%s\nsource\t%s\ntarget\t%s\nurl\t%s" % (title, pr_repo_desc, target_ref, url))
+ continue_maybe("Proceed with merging pull request #%s?" % pr_num)
+
+ merged_refs = [target_ref]
+
+ merge_hash = merge_pr(pr_num, target_ref, title, body, pr_repo_desc)
+
+ pick_prompt = "Would you like to pick %s into another branch?" % merge_hash
+ while input("\n%s (y/n): " % pick_prompt).lower() == "y":
+ merged_refs = merged_refs + [cherry_pick(pr_num, merge_hash, latest_branch)]
+
+ if JIRA_IMPORTED:
+ if ASF_USERNAME and ASF_PASSWORD:
+ continue_maybe("Would you like to update an associated JIRA?")
+ jira_comment = "Issue resolved by pull request %s\n[%s/%s]" % (
+ pr_num,
+ GITHUB_BASE,
+ pr_num,
+ )
+ resolve_jira_issues(title, merged_refs, jira_comment)
+ else:
+ print("ASF_USERNAME and ASF_PASSWORD not set")
+ print("Exiting without trying to close the associated JIRA.")
+ else:
+ print("Could not find jira-python library. Run 'sudo pip3 install jira' to install.")
+ print("Exiting without trying to close the associated JIRA.")
+
+
+if __name__ == "__main__":
+ import doctest
+
+ (failure_count, test_count) = doctest.testmod()
+ if failure_count:
+ sys.exit(-1)
+ try:
+ main()
+ except BaseException:
+ clean_up()
+ raise
diff --git a/requirements.txt b/requirements.txt
index 6cd16668c..b44103837 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -18,10 +18,12 @@
certifi==2022.9.24
charset-normalizer==2.1.1
click==8.1.3
+defusedxml==0.7.1
ghp-import==2.1.0
idna==3.4
importlib-metadata==4.12.0
Jinja2==3.1.2
+jira==3.5.1
Markdown==3.3.7
MarkupSafe==2.1.1
mergedeep==1.3.4
@@ -30,6 +32,7 @@ mkdocs-include-markdown-plugin==4.0.0
mkdocs-macros-plugin==0.7.0
mkdocs-material==8.5.11
mkdocs-material-extensions==1.1.1
+oauthlib==3.2.2
packaging==21.3
Pygments==2.12.0
pymdown-extensions==9.5
@@ -38,8 +41,11 @@ python-dateutil==2.8.2
PyYAML==6.0
pyyaml_env_tag==0.1
requests==2.28.1
+requests-oauthlib==1.3.1
+requests-toolbelt==1.0.0
six==1.16.0
termcolor==1.1.0
+typing_extensions==4.6.3
urllib3==1.26.13
watchdog==2.1.9
zipp==3.8.1
[incubator-celeborn] 01/02: [CELEBORN-105][TEST] Kubernetes Integration Test
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhouky pushed a commit to branch branch-0.3
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
commit 24222f2a20ede7250e3db92ae1af3d643e30d23a
Author: zwangsheng <22...@qq.com>
AuthorDate: Mon Jun 5 12:11:29 2023 +0800
[CELEBORN-105][TEST] Kubernetes Integration Test
---
.github/workflows/integration.yml | 78 ++++++++++++++++++++++
pom.xml | 14 ++++
tests/kubernetes-it/pom.xml | 70 +++++++++++++++++++
.../src/test/resources/log4j2-test.xml | 37 ++++++++++
.../DeploySuite.scala | 59 ++++++++++++++++
.../WithMiniKube.scala | 29 ++++++++
6 files changed, 287 insertions(+)
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
new file mode 100644
index 000000000..6f5c4f2e8
--- /dev/null
+++ b/.github/workflows/integration.yml
@@ -0,0 +1,78 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: Integration Test
+
+on:
+ push:
+ branches:
+ - main
+ - branch-*
+ pull_request:
+ branches:
+ - main
+ - branch-*
+env:
+ MINIKUBE_VERSION: v1.29.0
+ KUBERNETES_VERSION: v1.26.1
+
+jobs:
+ celeborn_integration_test:
+ runs-on: ubuntu-22.04
+ steps:
+ - uses: actions/checkout@v2
+ - uses: actions/setup-java@v2
+ name: Setup JDK with Maven
+ with:
+ distribution: zulu
+ java-version: 8
+ cache: maven
+ check-latest: false
+ - name: Build Celeborn Binary
+ run:
+ ./build/make-distribution.sh -Pspark-3.3 -Pgoogle-mirror
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v1
+ - name: Build Celeborn Docker Image
+ uses: docker/build-push-action@v2
+ with:
+ context: ./dist
+ file: ./docker/Dockerfile
+ load: true
+ tags: apache/celeborn:latest
+ - name: Setup Helm
+ uses: azure/setup-helm@v3
+ with:
+ version: v3.10.0
+ - name: Setup Minikube
+ run: |
+ # https://minikube.sigs.k8s.io/docs/start/
+ curl -LO https://github.com/kubernetes/minikube/releases/download/${MINIKUBE_VERSION}/minikube-linux-amd64
+ sudo install minikube-linux-amd64 /usr/local/bin/minikube
+ minikube start --cpus 2 --memory 4096 --kubernetes-version=${KUBERNETES_VERSION} --force
+ - name: Deploy Celeborn
+ run: |
+ # https://minikube.sigs.k8s.io/docs/handbook/pushing/#7-loading-directly-to-in-cluster-container-runtime
+ minikube image load apache/celeborn:latest
+ helm install celeborn charts/celeborn -f ./tests/kubernetes-it/docker/helm/values.yaml
+ - name: Wait Celeborn Ready
+ run: >-
+ ./build/mvn clean install
+ -Pkubernetes-it
+ -Pspark-3.3
+ -Dtest=none
+ -DwildcardSuites=org.apache.celeborn.integration.DeploySuite
diff --git a/pom.xml b/pom.xml
index 3c080daf9..85f614077 100644
--- a/pom.xml
+++ b/pom.xml
@@ -89,6 +89,7 @@
<roaringbitmap.version>0.9.32</roaringbitmap.version>
<snakeyaml.version>1.33</snakeyaml.version>
<zstd-jni.version>1.5.2-1</zstd-jni.version>
+ <kubernetes-client.version>6.7.0</kubernetes-client.version>
<shading.prefix>org.apache.celeborn.shaded</shading.prefix>
@@ -405,6 +406,12 @@
<version>${mockito-scalatest.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>io.fabric8</groupId>
+ <artifactId>kubernetes-client</artifactId>
+ <version>${kubernetes-client.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</dependencyManagement>
@@ -1134,5 +1141,12 @@
</plugins>
</build>
</profile>
+
+ <profile>
+ <id>kubernetes-it</id>
+ <modules>
+ <module>tests/kubernetes-it</module>
+ </modules>
+ </profile>
</profiles>
</project>
diff --git a/tests/kubernetes-it/pom.xml b/tests/kubernetes-it/pom.xml
new file mode 100644
index 000000000..2efbb905d
--- /dev/null
+++ b/tests/kubernetes-it/pom.xml
@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.celeborn</groupId>
+ <artifactId>celeborn-parent_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>kubernetes-it</artifactId>
+ <packaging>jar</packaging>
+ <name>Celeborn Kubernetes Integration Test</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.celeborn</groupId>
+ <artifactId>celeborn-common_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.celeborn</groupId>
+ <artifactId>celeborn-client_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.celeborn</groupId>
+ <artifactId>celeborn-client_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.celeborn</groupId>
+ <artifactId>celeborn-master_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.celeborn</groupId>
+ <artifactId>celeborn-worker_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>io.fabric8</groupId>
+ <artifactId>kubernetes-client</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/tests/kubernetes-it/src/test/resources/log4j2-test.xml b/tests/kubernetes-it/src/test/resources/log4j2-test.xml
new file mode 100644
index 000000000..9bcf7f263
--- /dev/null
+++ b/tests/kubernetes-it/src/test/resources/log4j2-test.xml
@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<Configuration status="INFO">
+ <Appenders>
+ <Console name="stdout" target="SYSTEM_OUT">
+ <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss,SSS} %p [%t] %c{1}: %m%n%ex"/>
+ <Filters>
+ <ThresholdFilter level="FATAL"/>
+ </Filters>
+ </Console>
+ <File name="file" fileName="target/unit-tests.log">
+ <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss,SSS} %p [%t] %c{1}: %m%n%ex"/>
+ </File>
+ </Appenders>
+ <Loggers>
+ <Root level="INFO">
+ <AppenderRef ref="stdout"/>
+ <AppenderRef ref="file"/>
+ </Root>
+ </Loggers>
+</Configuration>
diff --git a/tests/kubernetes-it/src/test/scala/org.apache.celeborn.integration/DeploySuite.scala b/tests/kubernetes-it/src/test/scala/org.apache.celeborn.integration/DeploySuite.scala
new file mode 100644
index 000000000..7d93d6d06
--- /dev/null
+++ b/tests/kubernetes-it/src/test/scala/org.apache.celeborn.integration/DeploySuite.scala
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.celeborn.integration
+
+import scala.language.postfixOps
+
+import org.scalatest.concurrent.Eventually.eventually
+import org.scalatest.concurrent.Waiters.{interval, timeout}
+import org.scalatest.time.SpanSugar.convertIntToGrainOfTime
+
+import org.apache.celeborn.CelebornFunSuite
+import org.apache.celeborn.client.WithShuffleClientSuite
+
+// TODO need add shuffle client test
+class DeploySuite extends CelebornFunSuite with WithMiniKube {
+ final val masterStatefulSet =
+ kubernetesClient.apps().statefulSets().withName("celeborn-master").get()
+ assert(masterStatefulSet != null)
+ final val workerStatefulSet =
+ kubernetesClient.apps().statefulSets().withName("celeborn-worker").get()
+ assert(workerStatefulSet != null)
+
+ test("Check master started") {
+ val masterPod = kubernetesClient.pods().withName("celeborn-master-0")
+ // wait for master ready
+ eventually(timeout(5 minutes), interval(10 seconds)) {
+ val log = masterPod.getLog(true)
+ assert(log.contains("Master started."))
+ }
+
+ assert(masterPod.get().getStatus.getPhase == "Running")
+ }
+
+ test("Check worker started") {
+ val workerPod = kubernetesClient.pods().withName("celeborn-worker-0")
+ // wait for worker ready
+ eventually(timeout(5 minutes), interval(10 seconds)) {
+ val log = workerPod.getLog(true)
+ assert(log.contains("Register worker successfully."))
+ }
+
+ assert(workerPod.get().getStatus.getPhase == "Running")
+ }
+}
diff --git a/tests/kubernetes-it/src/test/scala/org.apache.celeborn.integration/WithMiniKube.scala b/tests/kubernetes-it/src/test/scala/org.apache.celeborn.integration/WithMiniKube.scala
new file mode 100644
index 000000000..8676d7174
--- /dev/null
+++ b/tests/kubernetes-it/src/test/scala/org.apache.celeborn.integration/WithMiniKube.scala
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.celeborn.integration
+
+import io.fabric8.kubernetes.client.{KubernetesClient, KubernetesClientBuilder}
+import io.fabric8.kubernetes.client.Config
+
+trait WithMiniKube {
+
+ lazy val kubernetesClient: KubernetesClient =
+ new KubernetesClientBuilder()
+ .withConfig(Config.autoConfigure("minikube"))
+ .build()
+}