You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by ru...@apache.org on 2024/07/17 16:01:14 UTC
(superset) 03/03: script works!
This is an automated email from the ASF dual-hosted git repository.
rusackas pushed a commit to branch more-scraping
in repository https://gitbox.apache.org/repos/asf/superset.git
commit 26e4c52b13c77b95d38c07156993c0378d8dd580
Author: Evan Rusackas <ev...@rusackas.com>
AuthorDate: Fri Jul 12 08:54:48 2024 -0600
script works!
---
.github/workflows/get_star_ranking.py | 148 ++++++++++++++++++++++++++++++++++
1 file changed, 148 insertions(+)
diff --git a/.github/workflows/get_star_ranking.py b/.github/workflows/get_star_ranking.py
new file mode 100644
index 0000000000..e558d73685
--- /dev/null
+++ b/.github/workflows/get_star_ranking.py
@@ -0,0 +1,148 @@
+import csv
+import json
+import time
+from typing import Any, Dict, List, Optional, Tuple
+
+import requests
+from requests.exceptions import RequestException
+
+# Configuration
+GITHUB_API_URL = "https://api.github.com/graphql"
+GITHUB_TOKEN = "" # Replace with your actual token
+QUERY = """
+{
+ search(query: "stars:>1", type: REPOSITORY, first: 100, after: AFTER_CURSOR) {
+ edges {
+ cursor
+ node {
+ ... on Repository {
+ nameWithOwner
+ stargazers {
+ totalCount
+ }
+ watchers {
+ totalCount
+ }
+ openIssues: issues(states: OPEN) {
+ totalCount
+ }
+ discussions {
+ totalCount
+ }
+ }
+ }
+ }
+ }
+}
+"""
+HEADERS = {"Authorization": f"Bearer {GITHUB_TOKEN}"}
+TARGET_REPOSITORY: Optional[str] = None # Set this to None to print all repositories
+MAX_REPOS = 200 # Limit to the first 200 repositories
+MAX_RETRIES = 3
+RETRY_DELAY = 5 # seconds
+
+
+class RepositoryFetchError(Exception):
+ """Custom exception for repository fetching errors."""
+
+
+def fetch_repositories(after_cursor: Optional[str] = None) -> Dict[str, Any]:
+ query = QUERY.replace(
+ "AFTER_CURSOR", f'"{after_cursor}"' if after_cursor else "null"
+ )
+ for attempt in range(MAX_RETRIES):
+ try:
+ response = requests.post(
+ GITHUB_API_URL, json={"query": query}, headers=HEADERS, timeout=30
+ )
+ response.raise_for_status()
+ result = response.json()
+ if "errors" in result:
+ print(f"GraphQL errors: {json.dumps(result['errors'], indent=2)}")
+ raise RepositoryFetchError("GraphQL query returned errors")
+ if "data" not in result or "search" not in result["data"]:
+ print(f"Unexpected response structure: {json.dumps(result, indent=2)}")
+ raise RepositoryFetchError("Unexpected response structure")
+ return result["data"]["search"]
+ except RequestException as request_error:
+ print(f"Attempt {attempt + 1} failed: {str(request_error)}")
+ if attempt < MAX_RETRIES - 1:
+ print(f"Retrying in {RETRY_DELAY} seconds...")
+ time.sleep(RETRY_DELAY)
+ else:
+ print("Max retries reached. Exiting.")
+ raise RepositoryFetchError(
+ "Failed to fetch repositories after all retries"
+ ) from request_error
+ except json.JSONDecodeError as json_error:
+ print(f"Error decoding JSON response: {str(json_error)}")
+ if attempt < MAX_RETRIES - 1:
+ print(f"Retrying in {RETRY_DELAY} seconds...")
+ time.sleep(RETRY_DELAY)
+ else:
+ print("Max retries reached. Exiting.")
+ raise RepositoryFetchError(
+ "Failed to decode JSON response"
+ ) from json_error
+
+ raise RepositoryFetchError("Failed to fetch repositories after all retries")
+
+
+def main() -> None:
+ repositories: List[Tuple[int, str, int, int, int, int]] = []
+ after_cursor: Optional[str] = None
+ rank = 0
+ target_repo_found = False
+
+ while not target_repo_found and rank < MAX_REPOS:
+ try:
+ result = fetch_repositories(after_cursor)
+ except RepositoryFetchError as fetch_error:
+ print(f"Error fetching repositories: {str(fetch_error)}")
+ break
+
+ for edge in result["edges"]:
+ rank += 1
+ repo = edge["node"]
+ repo_name = repo["nameWithOwner"]
+ stars = repo["stargazers"]["totalCount"]
+ watchers = repo["watchers"]["totalCount"]
+ open_issues = repo["openIssues"]["totalCount"]
+ discussions = repo["discussions"]["totalCount"]
+
+ repositories.append(
+ (rank, repo_name, stars, watchers, open_issues, discussions)
+ )
+
+ if TARGET_REPOSITORY and repo_name == TARGET_REPOSITORY:
+ target_repo_found = True
+ print(
+ f"Found target repository: {repo_name} at rank {rank} with {stars} stars"
+ )
+ break
+
+ if rank >= MAX_REPOS:
+ break
+
+ if target_repo_found or not result["edges"] or rank >= MAX_REPOS:
+ break
+ after_cursor = result["edges"][-1]["cursor"]
+
+ # Save to CSV
+ with open("github_top_repositories.csv", "w", newline="") as csvfile:
+ csvwriter = csv.writer(csvfile)
+ csvwriter.writerow(
+ ["Rank", "Repository", "Stars", "Watchers", "Open Issues", "Discussions"]
+ )
+ csvwriter.writerows(repositories)
+
+ # Print all rows if TARGET_REPOSITORY is None or False
+ if not TARGET_REPOSITORY:
+ for repo in repositories:
+ print(
+ f"Rank: {repo[0]}, Repository: {repo[1]}, Stars: {repo[2]}, Watchers: {repo[3]}, Open Issues: {repo[4]}, Discussions: {repo[5]}"
+ )
+
+
+if __name__ == "__main__":
+ main()