You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by jo...@apache.org on 2022/04/12 14:59:19 UTC
[arrow] branch master updated: ARROW-16165: [CI][Archery] Fix nightly query to crossbow to send reports
This is an automated email from the ASF dual-hosted git repository.
jonkeane pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 8f5936f4bc ARROW-16165: [CI][Archery] Fix nightly query to crossbow to send reports
8f5936f4bc is described below
commit 8f5936f4bc5b02b3b3516831bbde23b55c213249
Author: Raúl Cumplido <ra...@gmail.com>
AuthorDate: Tue Apr 12 09:59:09 2022 -0500
ARROW-16165: [CI][Archery] Fix nightly query to crossbow to send reports
This PR fixes the current issue with our nightly reports as seen here: https://github.com/ursacomputing/crossbow/runs/5840285120?check_suite_focus=true)
The issue could be reproduced using the prefix that crossbow reports uses:
```
job_prefix=nightly-${{ inputs.report_type }}-$(date -I)
job_id=$(archery crossbow latest-prefix ${job_prefix})
```
Before the fix, when using the following query:
```
$ archery crossbow latest-prefix --no-fetch nightly-packaging-2022-04-10
Traceback (most recent call last):
File "/home/raulcd/open_source/pyarrow-dev/bin/archery", line 33, in <module>
sys.exit(load_entry_point('archery', 'console_scripts', 'archery')())
File "/home/raulcd/open_source/pyarrow-dev/lib/python3.10/site-packages/click/core.py", line 1130, in __call__
return self.main(*args, **kwargs)
File "/home/raulcd/open_source/pyarrow-dev/lib/python3.10/site-packages/click/core.py", line 1055, in main
rv = self.invoke(ctx)
File "/home/raulcd/open_source/pyarrow-dev/lib/python3.10/site-packages/click/core.py", line 1657, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/home/raulcd/open_source/pyarrow-dev/lib/python3.10/site-packages/click/core.py", line 1657, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/home/raulcd/open_source/pyarrow-dev/lib/python3.10/site-packages/click/core.py", line 1404, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/home/raulcd/open_source/pyarrow-dev/lib/python3.10/site-packages/click/core.py", line 760, in invoke
return __callback(*args, **kwargs)
File "/home/raulcd/open_source/pyarrow-dev/lib/python3.10/site-packages/click/decorators.py", line 38, in new_func
return f(get_current_context().obj, *args, **kwargs)
File "/home/raulcd/open_source/arrow/dev/archery/archery/crossbow/cli.py", line 237, in latest_prefix
latest = queue.latest_for_prefix(prefix)
File "/home/raulcd/open_source/arrow/dev/archery/archery/crossbow/core.py", line 568, in latest_for_prefix
latest_id += "-0"
TypeError: unsupported operand type(s) for +=: 'int' and 'str'
```
After the fix:
```
$ archery crossbow latest-prefix --no-fetch nightly-packaging-2022-04-10
nightly-packaging-2022-04-10-0
$ archery crossbow latest-prefix --no-fetch nightly-packaging
nightly-packaging-2022-04-11-0
```
Closes #12862 from raulcd/ARROW-16165
Authored-by: Raúl Cumplido <ra...@gmail.com>
Signed-off-by: Jonathan Keane <jk...@gmail.com>
---
dev/archery/archery/crossbow/core.py | 9 ++++++++-
dev/archery/archery/crossbow/tests/test_core.py | 26 ++++++++++++++++++++++++-
2 files changed, 33 insertions(+), 2 deletions(-)
diff --git a/dev/archery/archery/crossbow/core.py b/dev/archery/archery/crossbow/core.py
index c41582ad40..1ad4763e29 100644
--- a/dev/archery/archery/crossbow/core.py
+++ b/dev/archery/archery/crossbow/core.py
@@ -542,6 +542,12 @@ class Queue(Repo):
latest = -1
return latest
+ def _prefix_contains_date(self, prefix):
+ prefix_date_pattern = re.compile(r'[\w\/-]*-(\d+)-(\d+)-(\d+)')
+ match_prefix = prefix_date_pattern.match(prefix)
+ if match_prefix:
+ return match_prefix.group(0)[-10:]
+
def _latest_prefix_date(self, prefix):
pattern = re.compile(r'[\w\/-]*{}-(\d+)-(\d+)-(\d+)'.format(prefix))
matches = list(filter(None, map(pattern.match, self.repo.branches)))
@@ -559,7 +565,8 @@ class Queue(Repo):
return '{}-{}'.format(prefix, latest_id + 1)
def latest_for_prefix(self, prefix):
- if prefix.startswith("nightly"):
+ prefix_date = self._prefix_contains_date(prefix)
+ if prefix.startswith("nightly") and not prefix_date:
latest_id = self._latest_prefix_date(prefix)
if not latest_id:
raise RuntimeError(
diff --git a/dev/archery/archery/crossbow/tests/test_core.py b/dev/archery/archery/crossbow/tests/test_core.py
index 847aae2240..3d538b89b2 100644
--- a/dev/archery/archery/crossbow/tests/test_core.py
+++ b/dev/archery/archery/crossbow/tests/test_core.py
@@ -16,9 +16,10 @@
# under the License.
from archery.utils.source import ArrowSources
-from archery.crossbow import Config
+from archery.crossbow import Config, Queue
import pathlib
+from unittest import mock
def test_config():
@@ -69,3 +70,26 @@ def test_group_select_blocklist(request):
test_nightly_no_test_out = conf.select(groups=["nightly-no-test"])
assert test_nightly_no_test_out.keys(
) >= {"nightly-fine", "nightly-not-fine"}
+
+
+def test_latest_for_prefix(request):
+ queue = Queue(pathlib.Path(request.node.fspath).parent)
+ with mock.patch("archery.crossbow.core.Repo.repo") as mocked_repo:
+ mocked_repo.branches = [
+ "origin/nightly-packaging-2022-04-10-0",
+ "origin/nightly-packaging-2022-04-11-0",
+ ]
+ with mock.patch("archery.crossbow.core.Queue.get") as mocked_get:
+ queue.latest_for_prefix("nightly-packaging-2022-04-10")
+ mocked_get.assert_called_once_with(
+ "nightly-packaging-2022-04-10-0")
+
+ with mock.patch("archery.crossbow.core.Repo.repo") as mocked_repo:
+ mocked_repo.branches = [
+ "origin/nightly-packaging-2022-04-10-0",
+ "origin/nightly-packaging-2022-04-11-0",
+ ]
+ with mock.patch("archery.crossbow.core.Queue.get") as mocked_get:
+ queue.latest_for_prefix("nightly-packaging")
+ mocked_get.assert_called_once_with(
+ "nightly-packaging-2022-04-11-0")