You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by GitBox <gi...@apache.org> on 2018/08/09 18:09:12 UTC
[GitHub] CathyZhang0822 closed pull request #11957: [MXNET-691]Add LabelBot
functionality -- adding labels
CathyZhang0822 closed pull request #11957: [MXNET-691]Add LabelBot functionality -- adding labels
URL: https://github.com/apache/incubator-mxnet/pull/11957
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/mxnet-bot/LabelBotAddLabels/LabelBot.py b/mxnet-bot/LabelBotAddLabels/LabelBot.py
new file mode 100644
index 00000000000..ef156f1270a
--- /dev/null
+++ b/mxnet-bot/LabelBotAddLabels/LabelBot.py
@@ -0,0 +1,157 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import json
+import os
+from botocore.vendored import requests
+import re
+import logging
+import secret_manager
+logging.basicConfig(level=logging.INFO)
+
+class LabelBot:
+
+ def __init__(self,
+ repo=os.environ.get("repo"),
+ github_user=None,
+ github_oauth_token=None,
+ secret=True):
+ self.repo = repo
+ self.github_user = github_user
+ self.github_oauth_token = github_oauth_token
+ if secret:
+ self.get_secret()
+ self.auth = (self.github_user, self.github_oauth_token)
+ self.all_labels = None
+
+ def get_secret(self):
+ """
+ This method is to get secret value from Secrets Manager
+ """
+ secret = json.loads(secret_manager.get_secret())
+ self.github_user = secret["github_user"]
+ self.github_oauth_token = secret["github_oauth_token"]
+
+ def tokenize(self, string):
+ """
+ This method is to extract labels from comments
+ """
+ substring = string[string.find('[')+1: string.rfind(']')]
+ labels = [' '.join(label.split()) for label in substring.split(',')]
+ return labels
+
+ def clean_string(self, raw_string, sub_string):
+ """
+ This method is to convert all non-alphanumeric characters from raw_string to sub_string
+ """
+ cleans = re.sub("[^0-9a-zA-Z]", sub_string, raw_string)
+ return cleans.lower()
+
+ def count_pages(self, obj, state='all'):
+ """
+ This method is to count how many pages of issues/labels in total
+ obj could be "issues"/"labels"
+ state could be "open"/"closed"/"all", available to issues
+ """
+ assert obj in set(["issues", "labels"]), "Invalid Input!"
+ url = 'https://api.github.com/repos/{}/{}'.format(self.repo, obj)
+ if obj == 'issues':
+ response = requests.get(url, {'state': state}, auth=self.auth)
+ else:
+ response = requests.get(url, auth=self.auth)
+ assert response.status_code == 200, response.status_code
+ if "link" not in response.headers:
+ return 1
+ return int(self.clean_string(response.headers['link'], " ").split()[-3])
+
+ def find_notifications(self):
+ """
+ This method is to find comments which @mxnet-label-bot
+ """
+ data = []
+ pages = self.count_pages("issues")
+ for page in range(1, pages+1):
+ url = 'https://api.github.com/repos/' + self.repo + '/issues?page=' + str(page) \
+ + '&per_page=30'.format(repo=self.repo)
+ response = requests.get(url,
+ {'state': 'open',
+ 'base': 'master',
+ 'sort': 'created',
+ 'direction': 'desc'},
+ auth=self.auth)
+ for item in response.json():
+ # limit the amount of unlabeled issues per execution
+ if len(data) >= 50:
+ break
+ if "pull_request" in item:
+ continue
+ if not item['labels']:
+ if item['comments'] != 0:
+ labels = []
+ comments_url = "https://api.github.com/repos/{}/issues/{}/comments".format(self.repo,item['number'])
+ comments = requests.get(comments_url, auth=self.auth).json()
+ for comment in comments:
+ if "@mxnet-label-bot" in comment['body']:
+ labels += self.tokenize(comment['body'])
+ logging.info("issue: {}, comment: {}".format(str(item['number']),comment['body']))
+ if labels != []:
+ data.append({"issue": item['number'], "labels": labels})
+ return data
+
+ def find_all_labels(self):
+ """
+ This method is to find all existing labels in the repo
+ """
+ pages = self.count_pages("labels")
+ all_labels = []
+ for page in range(1, pages+1):
+ url = 'https://api.github.com/repos/' + self.repo + '/labels?page=' + str(page) \
+ + '&per_page=30'.format(repo=self.repo)
+ response = requests.get(url, auth=self.auth)
+ for item in response.json():
+ all_labels.append(item['name'].lower())
+ self.all_labels = set(all_labels)
+ return set(all_labels)
+
+ def add_github_labels(self, issue_num, labels):
+ """
+ This method is to add a list of labels to one issue.
+ First it will remove redundant white spaces from each label.
+ Then it will check whether labels exist in the repo.
+ At last, it will add existing labels to the issue
+ """
+ assert self.all_labels, "Find all labels first"
+ issue_labels_url = 'https://api.github.com/repos/{repo}/issues/{id}/labels'\
+ .format(repo=self.repo, id=issue_num)
+ # clean labels, remove duplicated spaces. ex: "hello world" -> "hello world"
+ labels = [" ".join(label.split()) for label in labels]
+ labels = [label for label in labels if label.lower() in self.all_labels]
+ response = requests.post(issue_labels_url, json.dumps(labels), auth=self.auth)
+ if response.status_code == 200:
+ logging.info('Successfully add labels to {}: {}.'.format(str(issue_num), str(labels)))
+ else:
+ logging.error("Could not add the label")
+ logging.error(response.json())
+
+ def label(self, issues):
+ """
+ This method is to add labels to multiple issues
+ Input is a json file: [{number:1, labels:[a,b]},{number:2, labels:[c,d]}]
+ """
+ self.find_all_labels()
+ for issue in issues:
+ self.add_github_labels(issue['issue'], issue['labels'])
+
diff --git a/mxnet-bot/LabelBotAddLabels/README.md b/mxnet-bot/LabelBotAddLabels/README.md
new file mode 100644
index 00000000000..4e04b1617a0
--- /dev/null
+++ b/mxnet-bot/LabelBotAddLabels/README.md
@@ -0,0 +1,55 @@
+# label bot
+This bot serves to help non-committers add labels to GitHub issues.
+
+"Hi @mxnet-label-bot, please add labels: [operator, feature request]"
+
+## Setup
+**Make sure that all settings is in the same *AWS region***
+#### 1. Store a secret
+*Create a secret using Terraform*
+* Configure ***variables.tf***
+ 1. In variable "github_credentials", fill in github_user and github_oauth_token. If you don't want to fill in here, then you can leave them blank.
+ After you set up a secret, you can go to [AWS Secrets Manager Console](https://console.aws.amazon.com/secretsmanager) console to manually fill in secret values.
+ 2. In variable "secret_name", fill in the name of your secret. ie:"github/credentials"
+* Run `terraform apply`. It will create the secret. Once setup, it will output the secret ARN. Write it down.
+ <div align="center">
+ <img src="https://s3-us-west-2.amazonaws.com/email-boy-images/Screen+Shot+2018-08-02+at+9.42.56+PM.png" ><br>
+ </div>
+
+
+#### 2. Deploy Lambda Function
+*Deploy this label bot using the serverless framework*
+* Configure ***severless.yml***
+ 1. Under ***iamRolesStatements***, replace ***Resource*** with the secret ARN
+ 2. Under ***environment***
+ 1. Set ***region_name*** as the same region of your secret.
+ 2. Replace ***secret_name*** with the secret name. (Same as the secret name you set in step1)
+ 3. Replace ***repo*** with the repo's name you want to test.
+* Deploy
+Open terminal, go to current directory. run
+```
+serverless deploy
+```
+Then it will set up those AWS services:
+* An IAM role for label bot with policies:
+```
+1.secretsmanager:ListSecrets
+2.secretsmanager:DescribeSecret
+3.secretsmanager:GetSecretValue
+4.cloudwatchlogs:CreateLogStream
+5.cloudwatchlogs:PutLogEvents
+```
+One thing to mention: this IAM role only has ***Read*** access to the secret created in step1.
+* A Lambda function will all code needed.
+* A CloudWatch event which will trigger the lambda function every 5 minutes.
+
+#### 3.Play with this bot
+* Go to the repo, under an **unlabeled** issue, comment "@mxnet-label-bot, please add labels:[bug]". One thing to mention, this bot can only add labels which **exist** in the repo.
+* Go to the lambda function's console, click **Test**.
+* Then labels will be added.
+ <div align="center">
+ <img src="https://s3-us-west-2.amazonaws.com/email-boy-images/Screen+Shot+2018-07-31+at+3.10.26+PM.png" width="600" height="150"><br>
+ </div>
+
+
+
diff --git a/mxnet-bot/LabelBotAddLabels/handler.py b/mxnet-bot/LabelBotAddLabels/handler.py
new file mode 100644
index 00000000000..0f83c866376
--- /dev/null
+++ b/mxnet-bot/LabelBotAddLabels/handler.py
@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# lambda handler
+from LabelBot import LabelBot
+
+def label_bot_lambda(event, context):
+ lb = LabelBot(secret=True)
+ data = lb.find_notifications()
+ lb.label(data)
+ return "Lambda is triggered successfully!"
+
diff --git a/mxnet-bot/LabelBotAddLabels/outputs.tf b/mxnet-bot/LabelBotAddLabels/outputs.tf
new file mode 100644
index 00000000000..2c182c9f14f
--- /dev/null
+++ b/mxnet-bot/LabelBotAddLabels/outputs.tf
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+output "secret_arn" {
+ value = "${aws_secretsmanager_secret.github_credentials.id}"
+}
diff --git a/mxnet-bot/LabelBotAddLabels/secret_manager.py b/mxnet-bot/LabelBotAddLabels/secret_manager.py
new file mode 100644
index 00000000000..6092564c878
--- /dev/null
+++ b/mxnet-bot/LabelBotAddLabels/secret_manager.py
@@ -0,0 +1,54 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This file is served to fetch secret from Secrets Manager
+import boto3
+from botocore.exceptions import ClientError
+import os
+import logging
+logging.basicConfig(level=logging.INFO)
+
+def get_secret():
+ '''
+ This method is to fetch secret values
+ Please configure secret_name and region_name as environment variables
+ '''
+ secret_name = os.environ.get("secret_name")
+ region_name = os.environ.get("region_name")
+ endpoint_url = "https://secretsmanager.{}.amazonaws.com".format(region_name)
+ session = boto3.session.Session()
+ client = session.client(
+ service_name='secretsmanager',
+ region_name=region_name,
+ endpoint_url=endpoint_url
+ )
+
+ try:
+ # Decrypted secret using the associated KMS CMK
+ # Depending on whether the secret was a string or binary, one of these fields will be populated
+ get_secret_value_response = client.get_secret_value(
+ SecretId=secret_name
+ )
+ if 'SecretString' in get_secret_value_response:
+ secret = get_secret_value_response['SecretString']
+ return secret
+ else:
+ binary_secret_data = get_secret_value_response['SecretBinary']
+ return binary_secret_data
+ except ClientError as e:
+ logging.exception(e.response['Error']['Code'])
+
diff --git a/mxnet-bot/LabelBotAddLabels/serverless.yml b/mxnet-bot/LabelBotAddLabels/serverless.yml
new file mode 100644
index 00000000000..c0d143326ca
--- /dev/null
+++ b/mxnet-bot/LabelBotAddLabels/serverless.yml
@@ -0,0 +1,57 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# Configurations
+
+service: LabelBot
+
+package:
+ exclude:
+ - ./**
+ include:
+ - LabelBot.py
+ - secret_manager.py
+ - handler.py
+
+provider:
+ name: aws
+ runtime: python3.6
+ timeout: 180
+ iamRoleStatements:
+ - Effect: "Allow"
+ Action:
+ - "secretsmanager:GetSecretValue"
+ - "secretsmanager:DescribeSecret"
+ # replace resource with your secret's ARN
+ Resource: "arn:aws:secretsmanager:{region_name}:{account_id}:secret:{secret_id}"
+ - Effect: "Allow"
+ Action:
+ - "secretsmanager:ListSecrets"
+ Resource: "*"
+
+functions:
+ label:
+ handler: handler.label_bot_lambda
+
+ events:
+ - schedule: rate(5 minutes)
+ environment:
+ # replace region_name with your secret's region
+ region_name : "us-east-1"
+ # replace secret_name with your secret's name
+ secret_name : "secret_name"
+ # replace REPO with "apache/incubator-mxnet"
+ repo : "repo_name"
diff --git a/mxnet-bot/LabelBotAddLabels/setup_secret.tf b/mxnet-bot/LabelBotAddLabels/setup_secret.tf
new file mode 100644
index 00000000000..ade63f5a735
--- /dev/null
+++ b/mxnet-bot/LabelBotAddLabels/setup_secret.tf
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+resource "aws_secretsmanager_secret" "github_credentials"{
+ name = "${var.secret_name}"
+}
+
+resource"aws_secretsmanager_secret_version" "github_credentials"{
+ secret_id = "${aws_secretsmanager_secret.github_credentials.id}"
+ secret_string = "${jsonencode(var.github_credentials)}"
+}
diff --git a/mxnet-bot/LabelBotAddLabels/variables.tf b/mxnet-bot/LabelBotAddLabels/variables.tf
new file mode 100644
index 00000000000..8f4d2a6d608
--- /dev/null
+++ b/mxnet-bot/LabelBotAddLabels/variables.tf
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+variable "github_credentials" {
+ default = {
+ github_user = ""
+ github_oauth_token = ""
+ }
+ type = "map"
+}
+
+variable "secret_name" {
+ default = ""
+}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services