You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@airflow.apache.org by GitBox <gi...@apache.org> on 2018/07/31 09:50:57 UTC
[GitHub] ashb closed pull request #3413: [AIRFLOW-2238] Switch PR tool to
push to Github
ashb closed pull request #3413: [AIRFLOW-2238] Switch PR tool to push to Github
URL: https://github.com/apache/incubator-airflow/pull/3413
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/dev/airflow-pr b/dev/airflow-pr
index 65243ddf87..08eb9aad36 100755
--- a/dev/airflow-pr
+++ b/dev/airflow-pr
@@ -54,7 +54,8 @@ except ImportError:
try:
import keyring
except ImportError:
- print("Could not find the keyring library. Run 'sudo pip install keyring' to install.")
+ print("Could not find the keyring library. "
+ "Run 'sudo pip install keyring' to install.")
sys.exit(-1)
# Location of your Airflow git development area
@@ -64,12 +65,12 @@ AIRFLOW_GIT_LOCATION = os.environ.get(
# Remote name which points to the Gihub site
GITHUB_REMOTE_NAME = os.environ.get("GITHUB_REMOTE_NAME", "github")
-# Remote name which points to Apache git
-APACHE_REMOTE_NAME = os.environ.get("APACHE_REMOTE_NAME", "apache")
-# OAuth key used for issuing requests against the GitHub API. If this is not defined, then requests
-# will be unauthenticated. You should only need to configure this if you find yourself regularly
-# exceeding your IP's unauthenticated request rate limit. You can create an OAuth key at
-# https://github.com/settings/tokens. This tool only requires the "public_repo" scope.
+# OAuth key used for issuing requests against the GitHub API. If this is not
+# defined, then requests will be unauthenticated. You should only need to
+# configure this if you find yourself regularly exceeding your IP's
+# unauthenticated request rate limit. You can create an OAuth key at
+# https://github.com/settings/tokens. This tool only requires the "public_repo"
+# scope.
GITHUB_OAUTH_KEY = os.environ.get("GITHUB_OAUTH_KEY")
GITHUB_BASE = "https://github.com/apache/incubator-airflow/pull"
@@ -172,7 +173,7 @@ def merge_pr(pr_num, target_ref, title, body, pr_repo_desc, local):
pr_branch_name = "%s_MERGE_PR_%s" % (BRANCH_PREFIX, pr_num)
target_branch_name = "%s_MERGE_PR_%s_%s" % (BRANCH_PREFIX, pr_num, target_ref.upper())
run_cmd("git fetch %s pull/%s/head:%s" % (GITHUB_REMOTE_NAME, pr_num, pr_branch_name))
- run_cmd("git fetch %s %s:%s" % (APACHE_REMOTE_NAME, target_ref, target_branch_name))
+ run_cmd("git fetch %s %s:%s" % (GITHUB_REMOTE_NAME, target_ref, target_branch_name))
run_cmd("git checkout %s" % target_branch_name)
had_conflicts = False
@@ -205,7 +206,8 @@ def merge_pr(pr_num, target_ref, title, body, pr_repo_desc, local):
except Exception as e:
msg = "Error merging: %s\nWould you like to manually fix-up this merge?" % e
continue_maybe(msg)
- msg = "Okay, please fix any conflicts and 'git add' conflicting files... Finished?"
+ msg = ("Okay, please fix any conflicts and 'git add' conflicting files... " +
+ "Finished?")
continue_maybe(msg)
had_conflicts = True
@@ -216,7 +218,6 @@ def merge_pr(pr_num, target_ref, title, body, pr_repo_desc, local):
if pr_commits:
all_text += ' '.join(c['commit']['message'] for c in pr_commits)
all_jira_refs = standardize_jira_ref(all_text, only_jira=True)
- all_jira_issues = re.findall("AIRFLOW-[0-9]{1,6}", all_jira_refs)
merge_message_flags = []
@@ -315,7 +316,6 @@ def merge_pr(pr_num, target_ref, title, body, pr_repo_desc, local):
if primary_author == "":
primary_author = distinct_authors[0]
- authors = "\n".join(["Author: %s" % a for a in distinct_authors])
merge_message_flags.append(u'--author="{}"'.format(primary_author))
else:
@@ -327,7 +327,7 @@ def merge_pr(pr_num, target_ref, title, body, pr_repo_desc, local):
# reflow commit message
seen_first_line = False
for i in range(1, len(merge_message_flags)):
- if merge_message_flags[i-1] == '-m':
+ if merge_message_flags[i - 1] == '-m':
# let the first line be as long as the user wants
if not seen_first_line:
if '\n\n' in merge_message_flags[i]:
@@ -376,7 +376,7 @@ def merge_pr(pr_num, target_ref, title, body, pr_repo_desc, local):
run_cmd(['git', 'commit'] + commit_flags, echo_cmd=False)
if local:
- msg ='\n' + reflow("""
+ msg = '\n' + reflow("""
The PR has been merged locally in branch {}.
You may leave this program running while you work on it. When
you are finished, press any key to delete the PR branch and
@@ -390,13 +390,13 @@ def merge_pr(pr_num, target_ref, title, body, pr_repo_desc, local):
else:
continue_maybe(
'\n\nThe local merge is complete ({}).\n'.format(
- target_branch_name)
- + click.style(
- 'Push to Apache ({})?'.format(APACHE_REMOTE_NAME), 'red'))
+ target_branch_name) +
+ click.style(
+ 'Push to Gitbox ({})?'.format(GITHUB_REMOTE_NAME), 'red'))
try:
run_cmd('git push %s %s:%s' % (
- APACHE_REMOTE_NAME, target_branch_name, target_ref))
+ GITHUB_REMOTE_NAME, target_branch_name, target_ref))
except Exception as e:
clean_up()
fail("Exception while pushing: %s" % e)
@@ -419,7 +419,7 @@ def cherry_pick(pr_num, merge_hash, default_branch):
BRANCH_PREFIX, pr_num, pick_ref.upper())
run_cmd("git fetch %s %s:%s" % (
- APACHE_REMOTE_NAME, pick_ref, pick_branch_name))
+ GITHUB_REMOTE_NAME, pick_ref, pick_branch_name))
run_cmd("git checkout %s" % pick_branch_name)
try:
@@ -435,12 +435,12 @@ def cherry_pick(pr_num, merge_hash, default_branch):
continue_maybe(msg)
continue_maybe("Pick complete (local ref %s). Push to %s?" % (
- pick_branch_name, APACHE_REMOTE_NAME))
+ pick_branch_name, GITHUB_REMOTE_NAME))
try:
run_cmd(
'git push %s %s:%s' % (
- APACHE_REMOTE_NAME, pick_branch_name, pick_ref))
+ GITHUB_REMOTE_NAME, pick_branch_name, pick_ref))
except Exception as e:
clean_up()
fail("Exception while pushing: %s" % e)
@@ -459,7 +459,7 @@ def fix_version_from_branch(branch, versions):
if branch == "master":
return versions[0]
else:
- #TODO adopt a release scheme with branches. Spark uses branch-XX.
+ # TODO adopt a release scheme with branches. Spark uses branch-XX.
branch_ver = branch.replace("branch-", "")
versions = list(filter(
lambda x: x.name.startswith(branch_ver), versions))
@@ -692,18 +692,19 @@ def resolve_jira_issue(comment=None, jira_id=None, merge_branches=None):
action["id"],
fixVersions=jira_fix_versions,
comment=comment or None,
- resolution = {'id': resolution.raw['id']})
+ resolution={'id': resolution.raw['id']})
click.echo("Successfully resolved {id}{fv}!".format(
id=jira_id,
fv=' with fix versions={}'.format(fix_versions) if fix_versions else ''
- ))
+ ))
def standardize_jira_ref(text, only_jira=False):
"""
Standardize the [AIRFLOW-XXXXX] [MODULE] prefix
- Converts "[AIRFLOW-XXX][mllib] Issue", "[MLLib] AIRFLOW-XXX. Issue" or "AIRFLOW XXX [MLLIB]: Issue" to "[AIRFLOW-XXX][MLLIB] Issue"
+ Converts "[AIRFLOW-XXX][mllib] Issue", "[MLLib] AIRFLOW-XXX. Issue" or
+ "AIRFLOW XXX [MLLIB]: Issue" to "[AIRFLOW-XXX][MLLIB] Issue"
>>> standardize_jira_ref("[AIRFLOW-5821] [SQL] ParquetRelation2 CTAS should check if delete is successful")
'[AIRFLOW-5821][SQL] ParquetRelation2 CTAS should check if delete is successful'
@@ -725,7 +726,7 @@ def standardize_jira_ref(text, only_jira=False):
'Additional information for users building from source code'
>>> standardize_jira_ref('AIRFLOW 35 AIRFLOW--36 AIRFLOW 37 test', only_jira=True)
'[AIRFLOW-35][AIRFLOW-36][AIRFLOW-37]'
- """
+ """ # noqa
jira_refs = []
components = []
@@ -770,8 +771,8 @@ def standardize_jira_ref(text, only_jira=False):
# Assemble full text (JIRA ref(s), module(s), remaining text)
clean_text = ''.join(unique(jira_refs)).strip()
if not only_jira:
- clean_text += (
- ''.join(unique(components)).strip() + " " + text.strip())
+ clean_text += (
+ ''.join(unique(components)).strip() + " " + text.strip())
# Replace multiple spaces with a single space, e.g. if no jira refs
# and/or components were included
@@ -806,9 +807,6 @@ def main(pr_num, local=False):
- GITHUB_REMOTE_NAME
GitHub remote name (defaults to "github")
- - APACHE_REMOTE_NAME
- Apache git remote name (defaults to "apache")
-
- JIRA_USERNAME
ASF JIRA username for automatically closing JIRA issues. Users will be
prompted if it is not set.
@@ -847,7 +845,6 @@ def main(pr_num, local=False):
click.echo('Working with pull request {}'.format(pr_num))
pr = get_json("{}/pulls/{}".format(GITHUB_API_BASE, pr_num))
- pr_events = get_json("{}/issues/{}/events".format(GITHUB_API_BASE, pr_num))
url = pr["url"]
@@ -875,33 +872,6 @@ def main(pr_num, local=False):
base_ref = pr["head"]["ref"]
pr_repo_desc = "%s/%s" % (user_login, base_ref)
- # Merged pull requests are either closed or merged by asfgit
- merge_commits = [
- e for e in pr_events
- if e["actor"]["login"] == GITHUB_USER and
- (e["event"] == "closed" or e["event"] == "merged")]
-
- if merge_commits and False:
- merge_hash = merge_commits[0]["commit_id"]
- message = get_json(
- "%s/commits/%s" % (GITHUB_API_BASE, merge_hash)
- )["commit"]["message"]
-
- continue_maybe(
- "Pull request %s has already been merged. "
- "Do you want to backport?" % pr_num)
- commit_is_downloaded = run_cmd(
- ['git', 'rev-parse', '--quiet', '--verify',
- "%s^{commit}" % merge_hash]) != ""
- if not commit_is_downloaded:
- fail(
- "Couldn't find any merge commit for #%s, "
- "you may need to update HEAD." % pr_num)
-
- click.echo("Found commit %s:\n%s" % (merge_hash, message))
- cherry_pick(pr_num, merge_hash, latest_branch)
- sys.exit(0)
-
if not bool(pr["mergeable"]):
msg = ('Pull request {} is not mergeable in its current form.\n'
'Continue anyway? (experts only!)'.format(pr_num))
@@ -961,11 +931,10 @@ def cli():
This tool should be used by Airflow committers to test PRs, merge them
into the master branch, and close related JIRA issues.
- Before you begin, make sure you have created the 'apache' and 'github' git
- remotes. You can use the "setup_git_remotes" command to do this
- automatically. If you do not want to use these remote names, you can tell
- the PR tool by setting the appropriate environment variables. For more
- information, run:
+ Before you begin, make sure you have created the 'github' git remote. You
+ can use the "setup_git_remotes" command to do this automatically. If you do
+ not want to use these remote names, you can tell the PR tool by setting the
+ appropriate environment variable. For more information, run:
airflow-pr merge --help
"""
@@ -986,11 +955,10 @@ def cli():
def merge(pr_num):
"""
Utility for creating well-formed pull request merges and pushing them
- to Apache, as well as closing JIRA issues.
+ to Gitbox (a.k.a. GitHub), as well as closing JIRA issues.
This tool assumes you already have a local Airflow git folder and that you
- have added remotes corresponding to both (i) the github apache Airflow
- mirror and (ii) the apache git repo.
+ have added remote corresponding to the github apache Airflow repo.
To configure the tool, set the following env vars:
- AIRFLOW_GIT
@@ -1000,9 +968,6 @@ def merge(pr_num):
- GITHUB_REMOTE_NAME
GitHub remote name (defaults to "github")
- - APACHE_REMOTE_NAME
- Apache git remote name (defaults to "apache")
-
- JIRA_USERNAME
ASF JIRA username for automatically closing JIRA issues. Users will be
prompted if it is not set.
@@ -1045,11 +1010,9 @@ def setup_git_remotes():
click.echo(reflow("""
This command will create git remotes to mirror the following
structure. If you do not want to use these names, you must set the
- GITHUB_REMOTE_NAME and APACHE_REMOTE_NAME environment variables:
+ GITHUB_REMOTE_NAME environment variable:
git remote -v
- apache https://git-wip-us.apache.org/repos/asf/incubator-airflow.git (fetch)
- apache https://git-wip-us.apache.org/repos/asf/incubator-airflow.git (push)
github https://github.com/apache/incubator-airflow.git (fetch)
github https://github.com/apache/incubator-airflow.git (push)
@@ -1058,13 +1021,6 @@ def setup_git_remotes():
continue_maybe('Do you want to continue?')
error = False
- try:
- run_cmd('git remote add apache https://git-wip-us.apache.org/repos/asf/incubator-airflow.git')
- except:
- click.echo(click.style(reflow(
- '>>ERROR: Could not create apache remote. If it already exists, '
- 'run `git remote remove apache` to delete it.', fg='red')))
- error = True
try:
run_cmd('git remote add github https://github.com/apache/incubator-airflow.git')
except:
@@ -1072,7 +1028,8 @@ def setup_git_remotes():
'>>ERROR: Could not create github remote. If it already exists, '
'run `git remote remove github` to delete it.', fg='red')))
error = True
- click.echo('Done setting up git remotes. Run git remote -v to see them.')
+ if not error:
+ click.echo('Done setting up git remotes. Run git remote -v to see them.')
if __name__ == "__main__":
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services