You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ap...@apache.org on 2025/09/25 14:03:14 UTC
(arrow) branch main updated: GH-47650: [Archery][Integration] Add option to generate gold files (#47651)
This is an automated email from the ASF dual-hosted git repository.
apitrou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 9a219256bf GH-47650: [Archery][Integration] Add option to generate gold files (#47651)
9a219256bf is described below
commit 9a219256bf68cbb1f5c1b9432c510de4ed860624
Author: Antoine Pitrou <an...@python.org>
AuthorDate: Thu Sep 25 16:03:08 2025 +0200
GH-47650: [Archery][Integration] Add option to generate gold files (#47651)
### Rationale for this change
Generating "gold" IPC files for integration/regression testing between Arrow implementations is currently a tedious (and undocumented) manual process.
### What changes are included in this PR?
Add an option to `archery integration` to automate generation of gold files in the right format, using the Arrow implementation selected on the command line.
### Are these changes tested?
Only manually.
### Are there any user-facing changes?
No.
* GitHub Issue: #47650
Authored-by: Antoine Pitrou <an...@python.org>
Signed-off-by: Antoine Pitrou <an...@python.org>
---
dev/archery/archery/cli.py | 28 +++++++++++++++-------------
dev/archery/archery/integration/datagen.py | 27 ++++++++++++++++++++++++---
dev/archery/archery/integration/runner.py | 22 ++++++++++++++--------
docs/source/format/Integration.rst | 17 +++++++++++++++++
4 files changed, 70 insertions(+), 24 deletions(-)
diff --git a/dev/archery/archery/cli.py b/dev/archery/archery/cli.py
index 7206e56df5..4bd20c5946 100644
--- a/dev/archery/archery/cli.py
+++ b/dev/archery/archery/cli.py
@@ -693,6 +693,8 @@ def _set_default(opt, default):
envvar="ARCHERY_INTEGRATION_TARGET_IMPLEMENTATIONS")
@click.option('--write-generated-json', default="",
help='Generate test JSON to indicated path')
+@click.option('--write-gold-files', default="",
+ help='Generate gold files to indicated path')
@click.option('--run-ipc', is_flag=True, default=False,
help='Run IPC integration tests')
@click.option('--run-flight', is_flag=True, default=False,
@@ -715,7 +717,7 @@ def _set_default(opt, default):
help=("Substring for test names to include in run, "
"e.g. -k primitive"))
def integration(with_all=False, random_seed=12345, write_generated_json="",
- **args):
+ write_gold_files="", **args):
"""If you don't specify the "--target-implementations" option nor
the "ARCHERY_INTEGRATION_TARGET_IMPLEMENTATIONS" environment
variable, test patterns are product of all specified
@@ -774,8 +776,9 @@ def integration(with_all=False, random_seed=12345, write_generated_json="",
"""
- from .integration.datagen import get_generated_json_files
- from .integration.runner import run_all_tests
+ from .integration.datagen import (
+ get_generated_json_files, generate_gold_files)
+ from .integration.runner import run_all_tests, select_testers
import numpy as np
# FIXME(bkietz) Include help strings for individual testers.
@@ -784,33 +787,32 @@ def integration(with_all=False, random_seed=12345, write_generated_json="",
# Make runs involving data generation deterministic
np.random.seed(random_seed)
- implementations = ['cpp', 'dotnet', 'java', 'js', 'go', 'nanoarrow', 'rust']
formats = ['ipc', 'flight', 'c_data']
- enabled_implementations = 0
- for lang in implementations:
- param = f'with_{lang}'
- if with_all:
- args[param] = with_all
- enabled_implementations += args[param]
-
enabled_formats = 0
for fmt in formats:
param = f'run_{fmt}'
enabled_formats += args[param]
+ testers, other_testers = select_testers(**args)
+
if write_generated_json:
os.makedirs(write_generated_json, exist_ok=True)
get_generated_json_files(tempdir=write_generated_json)
+ elif write_gold_files:
+ if len(testers) != 1 or len(other_testers) != 0:
+ raise click.UsageError(
+ "Need exactly one implementation to generate gold files; try --help")
+ generate_gold_files(testers[0], write_gold_files)
else:
if enabled_formats == 0:
raise click.UsageError(
"Need to enable at least one format to test "
"(IPC, Flight, C Data Interface); try --help")
- if enabled_implementations == 0:
+ if len(testers) == 0:
raise click.UsageError(
"Need to enable at least one implementation to test; try --help")
- run_all_tests(**args)
+ run_all_tests(testers, other_testers, **args)
@archery.command()
diff --git a/dev/archery/archery/integration/datagen.py b/dev/archery/archery/integration/datagen.py
index 5a60719ea6..1cbbd64d9e 100644
--- a/dev/archery/archery/integration/datagen.py
+++ b/dev/archery/archery/integration/datagen.py
@@ -17,10 +17,12 @@
from collections import namedtuple, OrderedDict
import binascii
+import gzip
import json
import os
import random
import tempfile
+import shutil
import numpy as np
@@ -1885,9 +1887,6 @@ def generate_extension_case():
def get_generated_json_files(tempdir=None):
tempdir = tempdir or tempfile.mkdtemp(prefix='arrow-integration-')
- def _temp_path():
- return
-
file_objs = [
generate_primitive_case([], name='primitive_no_batches'),
generate_primitive_case([17, 20], name='primitive'),
@@ -2003,3 +2002,25 @@ def get_generated_json_files(tempdir=None):
generated_paths.append(file_obj)
return generated_paths
+
+
+def generate_gold_files(tester, gold_dir):
+ os.makedirs(gold_dir, exist_ok=True)
+
+ # Generate JSON files
+ files = get_generated_json_files(gold_dir)
+ for f in files:
+ # For each JSON file, convert it to Arrow IPC file and stream
+ json_path = os.path.join(gold_dir, 'generated_' +
+ f.name + '.json')
+ arrow_file_path = os.path.join(gold_dir, 'generated_' +
+ f.name + '.arrow_file')
+ stream_path = os.path.join(gold_dir, 'generated_' +
+ f.name + '.stream')
+ tester.json_to_file(json_path, arrow_file_path)
+ tester.file_to_stream(arrow_file_path, stream_path)
+ # And GZip-compress the JSON file
+ with open(json_path, 'rb') as f_in:
+ with gzip.open(json_path + '.gz', 'wb') as f_out:
+ shutil.copyfileobj(f_in, f_out)
+ os.unlink(json_path)
diff --git a/dev/archery/archery/integration/runner.py b/dev/archery/archery/integration/runner.py
index 3c2c34f894..29e3364224 100644
--- a/dev/archery/archery/integration/runner.py
+++ b/dev/archery/archery/integration/runner.py
@@ -589,14 +589,12 @@ def get_static_json_files():
]
-def run_all_tests(with_cpp=True, with_java=True, with_js=True,
- with_dotnet=True, with_go=True, with_rust=False,
- with_nanoarrow=False, run_ipc=False, run_flight=False,
- run_c_data=False, tempdir=None, target_implementations="",
- **kwargs):
- tempdir = tempdir or tempfile.mkdtemp(prefix='arrow-integration-')
- target_implementations = \
- target_implementations.split(",") if target_implementations else []
+def select_testers(with_cpp=True, with_java=True, with_js=True,
+ with_dotnet=True, with_go=True, with_rust=False,
+ with_nanoarrow=False, target_implementations="",
+ **kwargs):
+ target_implementations = (target_implementations.split(",")
+ if target_implementations else [])
testers: List[Tester] = []
other_testers: List[Tester] = []
@@ -635,6 +633,14 @@ def run_all_tests(with_cpp=True, with_java=True, with_js=True,
from .tester_rust import RustTester
append_tester("rust", RustTester(**kwargs))
+ return testers, other_testers
+
+
+def run_all_tests(testers: List[Tester], other_testers: List[Tester],
+ run_ipc=False, run_flight=False, run_c_data=False,
+ tempdir=None, **kwargs):
+ tempdir = tempdir or tempfile.mkdtemp(prefix='arrow-integration-')
+
static_json_files = get_static_json_files()
generated_json_files = datagen.get_generated_json_files(tempdir=tempdir)
json_files = static_json_files + generated_json_files
diff --git a/docs/source/format/Integration.rst b/docs/source/format/Integration.rst
index 1d99773bfa..ca88a825dc 100644
--- a/docs/source/format/Integration.rst
+++ b/docs/source/format/Integration.rst
@@ -621,3 +621,20 @@ utility. Below are the test cases which are covered by them:
- ZSTD
* Batches with Shared Dictionaries
+
+Generating new Gold Files
+'''''''''''''''''''''''''
+
+From time to time, it is desirable to add new gold files, for example when the
+Columnar format or the IPC specification is update. Archery provides a dedicated
+option to do that.
+
+It is recommended to generate gold files using a well-known version of a Arrow
+implementation. For example, if a build of Arrow C++ exists in ``./build/release/``,
+one can generate new gold files in the ``/tmp/gold-files`` directory using the
+following command:
+
+.. code-block:: shell
+
+ export ARROW_CPP_EXE_PATH=./build/release/
+ archery integration --with-cpp 1 --write-gold-files=/tmp/gold-files