[lucene-jira-archive] 01/02: allow to change attachments download dirpath

[to...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

tomoko pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene-jira-archive.git

commit 4d04c861fb08ee635aa985036fa3adba909265b6
Author: Tomoko Uchida <to...@gmail.com>
AuthorDate: Sat Jul 2 09:39:49 2022 +0900

    allow to change attachments download dirpath
---
 migration/.env.example         |  3 ++-
 migration/.gitignore           |  1 -
 migration/src/common.py        |  5 +++--
 migration/src/download_jira.py | 10 +++++-----
 4 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/migration/.env.example b/migration/.env.example
index 842824a3..4b087674 100644
--- a/migration/.env.example
+++ b/migration/.env.example
@@ -1,4 +1,5 @@
 export GITHUB_PAT=
 export GITHUB_REPO=
 export GITHUB_ATT_REPO="apache/lucene-jira-archive"
-export GITHUB_ATT_BRANCH="main"
\ No newline at end of file
+export GITHUB_ATT_BRANCH="attachments"
+export ATTACHMENTS_DL_DIR=
\ No newline at end of file
diff --git a/migration/.gitignore b/migration/.gitignore
index a9d17ea0..0c0283b2 100644
--- a/migration/.gitignore
+++ b/migration/.gitignore
@@ -13,4 +13,3 @@ venv/
 .env
 
 log/
-attachments/
\ No newline at end of file
diff --git a/migration/src/common.py b/migration/src/common.py
index a6c373b1..93a6a02f 100644
--- a/migration/src/common.py
+++ b/migration/src/common.py
@@ -3,12 +3,13 @@ import logging
 from datetime import datetime
 import functools
 import time
-
+import os
+import tempfile
 
 LOG_DIRNAME = "log"
 
 JIRA_DUMP_DIRNAME = "jira-dump"
-JIRA_ATTACHMENTS_DIRNAME = "attachments"
+JIRA_ATTACHMENTS_DIRPATH = os.getenv("ATTACHMENTS_DL_DIR", str(Path(tempfile.gettempdir()).joinpath("attachments")))
 GITHUB_IMPORT_DATA_DIRNAME = "github-import-data"
 MAPPINGS_DATA_DIRNAME = "mappings-data"
 
diff --git a/migration/src/download_jira.py b/migration/src/download_jira.py
index 7c3a5e7f..92bebc32 100644
--- a/migration/src/download_jira.py
+++ b/migration/src/download_jira.py
@@ -13,7 +13,7 @@ from dataclasses import dataclass
 
 import requests
 
-from common import LOG_DIRNAME, JIRA_DUMP_DIRNAME, JIRA_ATTACHMENTS_DIRNAME, logging_setup, jira_dump_file, jira_attachments_dir, jira_issue_id
+from common import LOG_DIRNAME, JIRA_DUMP_DIRNAME, JIRA_ATTACHMENTS_DIRPATH, logging_setup, jira_dump_file, jira_attachments_dir, jira_issue_id
 
 log_dir = Path(__file__).resolve().parent.parent.joinpath(LOG_DIRNAME)
 logger = logging_setup(log_dir, "download_jira")
@@ -94,7 +94,7 @@ if __name__ == "__main__":
         dump_dir.mkdir()
     assert dump_dir.exists()
 
-    att_data_dir = Path(__file__).resolve().parent.parent.parent.joinpath(JIRA_ATTACHMENTS_DIRNAME)
+    att_data_dir = Path(JIRA_ATTACHMENTS_DIRPATH)
     if not att_data_dir.exists():
         att_data_dir.mkdir()
     assert att_data_dir.exists()
@@ -108,10 +108,10 @@ if __name__ == "__main__":
         else:
             issues.append(args.min)
     
-    logger.info(f"Downloading Jira issues in {dump_dir}")
+    logger.info(f"Downloading Jira issues in {dump_dir}. Attachments are saved in {att_data_dir}.")
     for num in issues:
-        download_issue(num, dump_dir)
-        download_attachments(num, dump_dir, att_data_dir)
+        if download_issue(num, dump_dir):
+            download_attachments(num, dump_dir, att_data_dir)
         time.sleep(DOWNLOAD_INTERVAL_SEC)
     
     logger.info("Done.")