You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by to...@apache.org on 2022/08/07 13:01:14 UTC
[lucene-jira-archive] branch show-patches-inline created (now ec8b56c3)
This is an automated email from the ASF dual-hosted git repository.
tomoko pushed a change to branch show-patches-inline
in repository https://gitbox.apache.org/repos/asf/lucene-jira-archive.git
at ec8b56c3 embed attachment data for known file types
This branch includes the following new commits:
new ec8b56c3 embed attachment data for known file types
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[lucene-jira-archive] 01/01: embed attachment data for known file types
Posted by to...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tomoko pushed a commit to branch show-patches-inline
in repository https://gitbox.apache.org/repos/asf/lucene-jira-archive.git
commit ec8b56c38df4ecc415526254278063f8e2041cf4
Author: Tomoko Uchida <to...@gmail.com>
AuthorDate: Sun Aug 7 22:01:05 2022 +0900
embed attachment data for known file types
---
migration/src/jira2github_import.py | 9 +++----
migration/src/jira_util.py | 47 ++++++++++++++++++++++++++++++++++++-
2 files changed, 51 insertions(+), 5 deletions(-)
diff --git a/migration/src/jira2github_import.py b/migration/src/jira2github_import.py
index 5ba69c03..cb61e72c 100644
--- a/migration/src/jira2github_import.py
+++ b/migration/src/jira2github_import.py
@@ -30,7 +30,7 @@ def jira_timestamp_to_github_timestamp(ts: str) -> str:
return ts[:-9] + "Z"
-def convert_issue(num: int, dump_dir: Path, output_dir: Path, account_map: dict[str, str], jira_users: dict[str, str], att_repo: str, att_branch: str, logger: Logger) -> bool:
+def convert_issue(num: int, dump_dir: Path, output_dir: Path, account_map: dict[str, str], jira_users: dict[str, str], att_repo: str, att_branch: str, att_dir: Optional[Path], logger: Logger) -> bool:
jira_id = jira_issue_id(num)
dump_file = jira_dump_file(dump_dir, num)
if not dump_file.exists():
@@ -101,7 +101,7 @@ def convert_issue(num: int, dump_dir: Path, output_dir: Path, account_map: dict[
resolutiondate_datetime = None
try:
- body = f'{convert_text(description, att_replace_map, account_map, jira_users)}\n\n'
+ body = f'{convert_text(description, att_replace_map, account_map, jira_users, att_dir)}\n\n'
for image_file in unmentioned_images:
# show orphaned (unmentioned) image files in the issue description
att_url = att_replace_map.get(image_file)
@@ -172,7 +172,7 @@ Migrated from [{jira_id}]({jira_issue_url(jira_id)}) by {reporter}"""
if comment_updated_datetime.date() != comment_created_datetime.date():
comment_time += f' [updated: {comment_updated_datetime.strftime("%b %d %Y")}]'
try:
- comment_body = f'{convert_text(comment_body, att_replace_map, account_map, jira_users)}\n\n'
+ comment_body = f'{convert_text(comment_body, att_replace_map, account_map, jira_users, att_dir)}\n\n'
# apply a special conversion for jira-bot's comments.
# see https://github.com/apache/lucene-jira-archive/issues/54
if comment_author_name == "jira-bot":
@@ -311,7 +311,8 @@ if __name__ == "__main__":
def task(num):
logger = logging.getLogger(name)
try:
- convert_issue(num, dump_dir, output_dir, account_map, jira_users, github_att_repo, github_att_branch, logger)
+ att_dir = Path(JIRA_ATTACHMENTS_DIRPATH).joinpath(jira_issue_id(num)) if JIRA_ATTACHMENTS_DIRPATH else None
+ convert_issue(num, dump_dir, output_dir, account_map, jira_users, github_att_repo, github_att_branch, att_dir, logger)
except Exception as e:
logger.error(traceback.format_exc(limit=100))
logger.error(f"Failed to convert Jira issue. An error '{str(e)}' occurred; skipped {jira_issue_id(num)}.")
diff --git a/migration/src/jira_util.py b/migration/src/jira_util.py
index d02c974f..46c250ce 100644
--- a/migration/src/jira_util.py
+++ b/migration/src/jira_util.py
@@ -1,3 +1,4 @@
+from pathlib import Path
import re
import itertools
from dataclasses import dataclass
@@ -236,6 +237,31 @@ REGEX_LINK = re.compile(r"\[([^\]]+)\]\(([^\)]+)\)")
REGEX_GITHUB_ISSUE_LINK = re.compile(r"(\s)(#\d+)(\s)")
+# common file extensions in Lucene Jira attachments
+FILE_EXT_TO_LANG = {
+ ".patch": "diff",
+ ".PATCH": "diff",
+ ".pat": "diff",
+ ".diff": "diff",
+ ".java": "java",
+ ".jj": "java",
+ ".jflex": "java", # text?
+ ".txt": "text",
+ ".log": "text",
+ ".out": "text",
+ ".alg": "text",
+ ".perf": "text",
+ ".benchmark": "text",
+ ".test": "text",
+ ".py": "python",
+ ".html": "html",
+ ".xml": "xml",
+ ".sh": "sh",
+ ".json": "json",
+ ".jsp": "jsp",
+ ".properties": "ini"
+}
+
def extract_embedded_image_files(text: str, image_files: list[str]) -> set[str]:
"""Extract embedded image files in the given text.
https://jira.atlassian.com/secure/WikiRendererHelpAction.jspa?section=images
@@ -253,7 +279,7 @@ def extract_embedded_image_files(text: str, image_files: list[str]) -> set[str]:
return embedded_image_files
-def convert_text(text: str, att_replace_map: dict[str, str] = {}, account_map: dict[str, str] = {}, jira_users: dict[str, str] = {}) -> str:
+def convert_text(text: str, att_replace_map: dict[str, str] = {}, account_map: dict[str, str] = {}, jira_users: dict[str, str] = {}, att_dir: Optional[Path] = None) -> str:
"""Convert Jira markup to Markdown
"""
def repl_att(m: re.Match):
@@ -318,6 +344,25 @@ def convert_text(text: str, att_replace_map: dict[str, str] = {}, account_map: d
# escape github style cross-issue link (#NN)
text = re.sub(REGEX_GITHUB_ISSUE_LINK, escape_gh_issue_link, text)
+ # embed attachments (patches, etc.) if possible
+ links = re.findall(REGEX_LINK, text)
+ if links and att_dir:
+ paths = list(filter(lambda p: p.exists, (att_dir.joinpath(x[0]) for x in links)))
+ if paths:
+ path = paths[0]
+ # skip unknown file extensions; skip too large files.
+ if path.suffix in FILE_EXT_TO_LANG and path.stat().st_size < 50000:
+ text += __textdata_as_details(path, FILE_EXT_TO_LANG[path.suffix])
+
+ return text
+
+
+def __textdata_as_details(path: Path, lang: str) -> str:
+ assert path.exists()
+ name = path.name
+ with open(path) as fp:
+ data = fp.read()
+ text = f"\n<details>\n<summary>{name}</summary>\n\n```{lang}\n{data}\n```\n\n</details>\n\n"
return text