You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ne...@apache.org on 2022/02/22 09:21:00 UTC
[arrow-rs] branch master updated: Arrow Rust + Conbench Integration (#1289)
This is an automated email from the ASF dual-hosted git repository.
nevime pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/master by this push:
new 4b89f7e Arrow Rust + Conbench Integration (#1289)
4b89f7e is described below
commit 4b89f7ee3549c24fa5997056b16a1cde60ce7043
Author: diana <di...@gmail.com>
AuthorDate: Tue Feb 22 02:20:54 2022 -0700
Arrow Rust + Conbench Integration (#1289)
* Arrow Rust + Conbench Integration
* remove --src-dir
---
conbench/.flake8 | 2 +
conbench/.gitignore | 130 ++++++++++++++++++++
conbench/.isort.cfg | 2 +
conbench/README.md | 251 ++++++++++++++++++++++++++++++++++++++
conbench/_criterion.py | 98 +++++++++++++++
conbench/benchmarks.json | 8 ++
conbench/benchmarks.py | 41 +++++++
conbench/requirements-test.txt | 3 +
conbench/requirements.txt | 1 +
dev/release/rat_exclude_files.txt | 5 +
10 files changed, 541 insertions(+)
diff --git a/conbench/.flake8 b/conbench/.flake8
new file mode 100644
index 0000000..e44b810
--- /dev/null
+++ b/conbench/.flake8
@@ -0,0 +1,2 @@
+[flake8]
+ignore = E501
diff --git a/conbench/.gitignore b/conbench/.gitignore
new file mode 100755
index 0000000..aa44ee2
--- /dev/null
+++ b/conbench/.gitignore
@@ -0,0 +1,130 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+# However, in case of collaboration, if having platform-specific dependencies or dependencies
+# having no cross-platform support, pipenv may install dependencies that don't work, or not
+# install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
diff --git a/conbench/.isort.cfg b/conbench/.isort.cfg
new file mode 100644
index 0000000..f238bf7
--- /dev/null
+++ b/conbench/.isort.cfg
@@ -0,0 +1,2 @@
+[settings]
+profile = black
diff --git a/conbench/README.md b/conbench/README.md
new file mode 100644
index 0000000..8c7f38c
--- /dev/null
+++ b/conbench/README.md
@@ -0,0 +1,251 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+# Arrow Rust + Conbench Integration
+
+
+## Quick start
+
+```
+$ cd ~/arrow-rs/conbench/
+$ conda create -y -n conbench python=3.9
+$ conda activate conbench
+(conbench) $ pip install -r requirements.txt
+(conbench) $ conbench arrow-rs
+```
+
+## Example output
+
+```
+{
+ "batch_id": "b68c559358cc43a3aab02d893d2693f4",
+ "context": {
+ "benchmark_language": "Rust"
+ },
+ "github": {
+ "commit": "ca33a0a50494f95840ade2e9509c3c3d4df35249",
+ "repository": "https://github.com/dianaclarke/arrow-rs"
+ },
+ "info": {},
+ "machine_info": {
+ "architecture_name": "x86_64",
+ "cpu_core_count": "8",
+ "cpu_frequency_max_hz": "2400000000",
+ "cpu_l1d_cache_bytes": "65536",
+ "cpu_l1i_cache_bytes": "131072",
+ "cpu_l2_cache_bytes": "4194304",
+ "cpu_l3_cache_bytes": "0",
+ "cpu_model_name": "Apple M1",
+ "cpu_thread_count": "8",
+ "gpu_count": "0",
+ "gpu_product_names": [],
+ "kernel_name": "20.6.0",
+ "memory_bytes": "17179869184",
+ "name": "diana",
+ "os_name": "macOS",
+ "os_version": "10.16"
+ },
+ "run_id": "08353595bde147fb9deebdb4facd019a",
+ "stats": {
+ "data": [
+ "0.000287",
+ "0.000286",
+ "0.000285",
+ "0.000281",
+ "0.000282",
+ "0.000277",
+ "0.000277",
+ "0.000286",
+ "0.000279",
+ "0.000282",
+ "0.000277",
+ "0.000277",
+ "0.000282",
+ "0.000276",
+ "0.000281",
+ "0.000281",
+ "0.000281",
+ "0.000281",
+ "0.000281",
+ "0.000284",
+ "0.000288",
+ "0.000278",
+ "0.000276",
+ "0.000278",
+ "0.000275",
+ "0.000275",
+ "0.000275",
+ "0.000275",
+ "0.000281",
+ "0.000284",
+ "0.000277",
+ "0.000277",
+ "0.000278",
+ "0.000282",
+ "0.000281",
+ "0.000284",
+ "0.000282",
+ "0.000279",
+ "0.000280",
+ "0.000281",
+ "0.000281",
+ "0.000286",
+ "0.000278",
+ "0.000278",
+ "0.000281",
+ "0.000276",
+ "0.000284",
+ "0.000281",
+ "0.000276",
+ "0.000276",
+ "0.000279",
+ "0.000283",
+ "0.000282",
+ "0.000278",
+ "0.000281",
+ "0.000284",
+ "0.000279",
+ "0.000276",
+ "0.000278",
+ "0.000283",
+ "0.000282",
+ "0.000276",
+ "0.000281",
+ "0.000279",
+ "0.000276",
+ "0.000277",
+ "0.000283",
+ "0.000279",
+ "0.000281",
+ "0.000283",
+ "0.000279",
+ "0.000282",
+ "0.000283",
+ "0.000278",
+ "0.000281",
+ "0.000282",
+ "0.000278",
+ "0.000276",
+ "0.000281",
+ "0.000278",
+ "0.000276",
+ "0.000282",
+ "0.000281",
+ "0.000282",
+ "0.000280",
+ "0.000281",
+ "0.000282",
+ "0.000280",
+ "0.000282",
+ "0.000280",
+ "0.000280",
+ "0.000282",
+ "0.000278",
+ "0.000284",
+ "0.000290",
+ "0.000282",
+ "0.000281",
+ "0.000281",
+ "0.000281",
+ "0.000278"
+ ],
+ "iqr": "0.000004",
+ "iterations": 100,
+ "max": "0.000290",
+ "mean": "0.000280",
+ "median": "0.000281",
+ "min": "0.000275",
+ "q1": "0.000278",
+ "q3": "0.000282",
+ "stdev": "0.000003",
+ "time_unit": "s",
+ "times": [],
+ "unit": "s"
+ },
+ "tags": {
+ "name": "nlike_utf8 scalar starts with",
+ "suite": "nlike_utf8 scalar starts with"
+ },
+ "timestamp": "2022-02-09T02:33:26.792404+00:00"
+}
+```
+
+## Debug with test benchmark
+
+```
+(conbench) $ cd ~/arrow-rs/conbench/
+(conbench) $ conbench test --iterations=3
+
+Benchmark result:
+{
+ "batch_id": "f4235d547e9d4f94925b54692e625d7d",
+ "context": {
+ "benchmark_language": "Python"
+ },
+ "github": {
+ "commit": "35e16be01e680e9381b2d1393c2e3f8e7acb7b13",
+ "repository": "https://github.com/dianaclarke/arrow-rs"
+ },
+ "info": {
+ "benchmark_language_version": "Python 3.9.7"
+ },
+ "machine_info": {
+ "architecture_name": "x86_64",
+ "cpu_core_count": "8",
+ "cpu_frequency_max_hz": "2400000000",
+ "cpu_l1d_cache_bytes": "65536",
+ "cpu_l1i_cache_bytes": "131072",
+ "cpu_l2_cache_bytes": "4194304",
+ "cpu_l3_cache_bytes": "0",
+ "cpu_model_name": "Apple M1",
+ "cpu_thread_count": "8",
+ "gpu_count": "0",
+ "gpu_product_names": [],
+ "kernel_name": "20.6.0",
+ "memory_bytes": "17179869184",
+ "name": "diana",
+ "os_name": "macOS",
+ "os_version": "10.16"
+ },
+ "run_id": "b2ca0d581cf14f21936276ff7ca5a940",
+ "stats": {
+ "data": [
+ "0.000002",
+ "0.000001",
+ "0.000001"
+ ],
+ "iqr": "0.000001",
+ "iterations": 3,
+ "max": "0.000002",
+ "mean": "0.000001",
+ "median": "0.000001",
+ "min": "0.000001",
+ "q1": "0.000001",
+ "q3": "0.000002",
+ "stdev": "0.000001",
+ "time_unit": "s",
+ "times": [],
+ "unit": "s"
+ },
+ "tags": {
+ "name": "test"
+ },
+ "timestamp": "2022-02-09T01:55:52.250727+00:00"
+}
+```
diff --git a/conbench/_criterion.py b/conbench/_criterion.py
new file mode 100644
index 0000000..168a1b9
--- /dev/null
+++ b/conbench/_criterion.py
@@ -0,0 +1,98 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import collections
+import csv
+import os
+import pathlib
+import subprocess
+
+import conbench.runner
+from conbench.machine_info import github_info
+
+
+def _result_in_seconds(row):
+ # sample_measured_value - The value of the measurement for this sample.
+ # Note that this is the measured value for the whole sample, not the
+ # time-per-iteration To calculate the time-per-iteration, use
+ # sample_measured_value/iteration_count
+ # -- https://bheisler.github.io/criterion.rs/book/user_guide/csv_output.html
+ count = int(row["iteration_count"])
+ sample = float(row["sample_measured_value"])
+ return sample / count / 10**9
+
+
+def _parse_benchmark_group(row):
+ parts = row["group"].split(",")
+ if len(parts) > 1:
+ suite, name = parts[0], ",".join(parts[1:])
+ else:
+ suite, name = row["group"], row["group"]
+ return suite.strip(), name.strip()
+
+
+def _read_results(src_dir):
+ results = collections.defaultdict(lambda: collections.defaultdict(list))
+ path = pathlib.Path(os.path.join(src_dir, "target", "criterion"))
+ for path in list(path.glob("**/new/raw.csv")):
+ with open(path) as csv_file:
+ reader = csv.DictReader(csv_file)
+ for row in reader:
+ suite, name = _parse_benchmark_group(row)
+ results[suite][name].append(_result_in_seconds(row))
+ return results
+
+
+def _execute_command(command):
+ try:
+ print(command)
+ result = subprocess.run(command, capture_output=True, check=True)
+ except subprocess.CalledProcessError as e:
+ print(e.stderr.decode("utf-8"))
+ raise e
+ return result.stdout.decode("utf-8"), result.stderr.decode("utf-8")
+
+
+class CriterionBenchmark(conbench.runner.Benchmark):
+ external = True
+
+ def run(self, **kwargs):
+ src_dir = os.path.join(os.getcwd(), "..")
+ self._cargo_bench(src_dir)
+ results = _read_results(src_dir)
+ for suite in results:
+ self.conbench.mark_new_batch()
+ for name, data in results[suite].items():
+ yield self._record_result(suite, name, data, kwargs)
+
+ def _cargo_bench(self, src_dir):
+ os.chdir(src_dir)
+ _execute_command(["cargo", "bench"])
+
+ def _record_result(self, suite, name, data, options):
+ tags = {"suite": suite}
+ result = {"data": data, "unit": "s"}
+ context = {"benchmark_language": "Rust"}
+ github = github_info()
+ return self.conbench.record(
+ result,
+ name,
+ tags=tags,
+ context=context,
+ github=github,
+ options=options,
+ )
diff --git a/conbench/benchmarks.json b/conbench/benchmarks.json
new file mode 100644
index 0000000..3e1f33a
--- /dev/null
+++ b/conbench/benchmarks.json
@@ -0,0 +1,8 @@
+[
+ {
+ "command": "arrow-rs",
+ "flags": {
+ "language": "Rust"
+ }
+ }
+]
diff --git a/conbench/benchmarks.py b/conbench/benchmarks.py
new file mode 100644
index 0000000..bc4c179
--- /dev/null
+++ b/conbench/benchmarks.py
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import conbench.runner
+
+import _criterion
+
+
+@conbench.runner.register_benchmark
+class TestBenchmark(conbench.runner.Benchmark):
+ name = "test"
+
+ def run(self, **kwargs):
+ yield self.conbench.benchmark(
+ self._f(),
+ self.name,
+ options=kwargs,
+ )
+
+ def _f(self):
+ return lambda: 1 + 1
+
+
+@conbench.runner.register_benchmark
+class CargoBenchmarks(_criterion.CriterionBenchmark):
+ name = "arrow-rs"
+ description = "Run Arrow Rust micro benchmarks."
diff --git a/conbench/requirements-test.txt b/conbench/requirements-test.txt
new file mode 100644
index 0000000..5e5647a
--- /dev/null
+++ b/conbench/requirements-test.txt
@@ -0,0 +1,3 @@
+black
+flake8
+isort
diff --git a/conbench/requirements.txt b/conbench/requirements.txt
new file mode 100644
index 0000000..a877c7b
--- /dev/null
+++ b/conbench/requirements.txt
@@ -0,0 +1 @@
+conbench
diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt
index c976e2c..92b1108 100644
--- a/dev/release/rat_exclude_files.txt
+++ b/dev/release/rat_exclude_files.txt
@@ -11,6 +11,11 @@ parquet_derive/test/dependency/*
Cargo.lock
filtered_rat.txt
rat.txt
+conbench/benchmarks.json
+conbench/requirements.txt
+conbench/requirements-test.txt
+conbench/.flake8
+conbench/.isort.cfg
# auto-generated
arrow-flight/src/arrow.flight.protocol.rs
.github/*