You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by to...@apache.org on 2022/08/07 13:01:15 UTC
[lucene-jira-archive] 01/01: embed attachment data for known file types
This is an automated email from the ASF dual-hosted git repository.
tomoko pushed a commit to branch show-patches-inline
in repository https://gitbox.apache.org/repos/asf/lucene-jira-archive.git
commit ec8b56c38df4ecc415526254278063f8e2041cf4
Author: Tomoko Uchida <to...@gmail.com>
AuthorDate: Sun Aug 7 22:01:05 2022 +0900
embed attachment data for known file types
---
migration/src/jira2github_import.py | 9 +++----
migration/src/jira_util.py | 47 ++++++++++++++++++++++++++++++++++++-
2 files changed, 51 insertions(+), 5 deletions(-)
diff --git a/migration/src/jira2github_import.py b/migration/src/jira2github_import.py
index 5ba69c03..cb61e72c 100644
--- a/migration/src/jira2github_import.py
+++ b/migration/src/jira2github_import.py
@@ -30,7 +30,7 @@ def jira_timestamp_to_github_timestamp(ts: str) -> str:
return ts[:-9] + "Z"
-def convert_issue(num: int, dump_dir: Path, output_dir: Path, account_map: dict[str, str], jira_users: dict[str, str], att_repo: str, att_branch: str, logger: Logger) -> bool:
+def convert_issue(num: int, dump_dir: Path, output_dir: Path, account_map: dict[str, str], jira_users: dict[str, str], att_repo: str, att_branch: str, att_dir: Optional[Path], logger: Logger) -> bool:
jira_id = jira_issue_id(num)
dump_file = jira_dump_file(dump_dir, num)
if not dump_file.exists():
@@ -101,7 +101,7 @@ def convert_issue(num: int, dump_dir: Path, output_dir: Path, account_map: dict[
resolutiondate_datetime = None
try:
- body = f'{convert_text(description, att_replace_map, account_map, jira_users)}\n\n'
+ body = f'{convert_text(description, att_replace_map, account_map, jira_users, att_dir)}\n\n'
for image_file in unmentioned_images:
# show orphaned (unmentioned) image files in the issue description
att_url = att_replace_map.get(image_file)
@@ -172,7 +172,7 @@ Migrated from [{jira_id}]({jira_issue_url(jira_id)}) by {reporter}"""
if comment_updated_datetime.date() != comment_created_datetime.date():
comment_time += f' [updated: {comment_updated_datetime.strftime("%b %d %Y")}]'
try:
- comment_body = f'{convert_text(comment_body, att_replace_map, account_map, jira_users)}\n\n'
+ comment_body = f'{convert_text(comment_body, att_replace_map, account_map, jira_users, att_dir)}\n\n'
# apply a special conversion for jira-bot's comments.
# see https://github.com/apache/lucene-jira-archive/issues/54
if comment_author_name == "jira-bot":
@@ -311,7 +311,8 @@ if __name__ == "__main__":
def task(num):
logger = logging.getLogger(name)
try:
- convert_issue(num, dump_dir, output_dir, account_map, jira_users, github_att_repo, github_att_branch, logger)
+ att_dir = Path(JIRA_ATTACHMENTS_DIRPATH).joinpath(jira_issue_id(num)) if JIRA_ATTACHMENTS_DIRPATH else None
+ convert_issue(num, dump_dir, output_dir, account_map, jira_users, github_att_repo, github_att_branch, att_dir, logger)
except Exception as e:
logger.error(traceback.format_exc(limit=100))
logger.error(f"Failed to convert Jira issue. An error '{str(e)}' occurred; skipped {jira_issue_id(num)}.")
diff --git a/migration/src/jira_util.py b/migration/src/jira_util.py
index d02c974f..46c250ce 100644
--- a/migration/src/jira_util.py
+++ b/migration/src/jira_util.py
@@ -1,3 +1,4 @@
+from pathlib import Path
import re
import itertools
from dataclasses import dataclass
@@ -236,6 +237,31 @@ REGEX_LINK = re.compile(r"\[([^\]]+)\]\(([^\)]+)\)")
REGEX_GITHUB_ISSUE_LINK = re.compile(r"(\s)(#\d+)(\s)")
+# common file extensions in Lucene Jira attachments
+FILE_EXT_TO_LANG = {
+ ".patch": "diff",
+ ".PATCH": "diff",
+ ".pat": "diff",
+ ".diff": "diff",
+ ".java": "java",
+ ".jj": "java",
+ ".jflex": "java", # text?
+ ".txt": "text",
+ ".log": "text",
+ ".out": "text",
+ ".alg": "text",
+ ".perf": "text",
+ ".benchmark": "text",
+ ".test": "text",
+ ".py": "python",
+ ".html": "html",
+ ".xml": "xml",
+ ".sh": "sh",
+ ".json": "json",
+ ".jsp": "jsp",
+ ".properties": "ini"
+}
+
def extract_embedded_image_files(text: str, image_files: list[str]) -> set[str]:
"""Extract embedded image files in the given text.
https://jira.atlassian.com/secure/WikiRendererHelpAction.jspa?section=images
@@ -253,7 +279,7 @@ def extract_embedded_image_files(text: str, image_files: list[str]) -> set[str]:
return embedded_image_files
-def convert_text(text: str, att_replace_map: dict[str, str] = {}, account_map: dict[str, str] = {}, jira_users: dict[str, str] = {}) -> str:
+def convert_text(text: str, att_replace_map: dict[str, str] = {}, account_map: dict[str, str] = {}, jira_users: dict[str, str] = {}, att_dir: Optional[Path] = None) -> str:
"""Convert Jira markup to Markdown
"""
def repl_att(m: re.Match):
@@ -318,6 +344,25 @@ def convert_text(text: str, att_replace_map: dict[str, str] = {}, account_map: d
# escape github style cross-issue link (#NN)
text = re.sub(REGEX_GITHUB_ISSUE_LINK, escape_gh_issue_link, text)
+ # embed attachments (patches, etc.) if possible
+ links = re.findall(REGEX_LINK, text)
+ if links and att_dir:
+ paths = list(filter(lambda p: p.exists, (att_dir.joinpath(x[0]) for x in links)))
+ if paths:
+ path = paths[0]
+ # skip unknown file extensions; skip too large files.
+ if path.suffix in FILE_EXT_TO_LANG and path.stat().st_size < 50000:
+ text += __textdata_as_details(path, FILE_EXT_TO_LANG[path.suffix])
+
+ return text
+
+
+def __textdata_as_details(path: Path, lang: str) -> str:
+ assert path.exists()
+ name = path.name
+ with open(path) as fp:
+ data = fp.read()
+ text = f"\n<details>\n<summary>{name}</summary>\n\n```{lang}\n{data}\n```\n\n</details>\n\n"
return text