You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2018/08/27 18:37:29 UTC
[arrow] branch master updated: ARROW-3125: [Python] Update ASV
instructions
This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new d20147e ARROW-3125: [Python] Update ASV instructions
d20147e is described below
commit d20147e7e3f506f03de9002da3f65467500869dc
Author: Antoine Pitrou <an...@python.org>
AuthorDate: Mon Aug 27 14:37:18 2018 -0400
ARROW-3125: [Python] Update ASV instructions
The ability to define custom install / build / uninstall commands was added in mainline ASV in https://github.com/airspeed-velocity/asv/pull/699
We don't need to use our own fork / PR anymore.
Author: Antoine Pitrou <an...@python.org>
Closes #2480 from pitrou/ARROW-3125-asv-instructions-update and squashes the following commits:
59ddc4e4 <Antoine Pitrou> ARROW-3125: Update ASV instructions
---
python/README-benchmarks.md | 4 ++--
python/asv-build.sh | 13 +++++++++----
python/asv-install.sh | 2 +-
python/asv-uninstall.sh | 2 +-
python/asv.conf.json | 10 +++++-----
python/benchmarks/parquet.py | 8 +++++++-
6 files changed, 25 insertions(+), 14 deletions(-)
diff --git a/python/README-benchmarks.md b/python/README-benchmarks.md
index c51d36e..77901f3 100644
--- a/python/README-benchmarks.md
+++ b/python/README-benchmarks.md
@@ -32,10 +32,10 @@ use your current Python interpreter and environment.
## Running with arbitrary revisions
ASV allows to store results and generate graphs of the benchmarks over
-the project's evolution. For this you have to install our fork of ASV:
+the project's evolution. For this you have the latest development version of ASV:
```shell
-pip install git+https://github.com/pitrou/asv.git@customize_commands
+pip install git+https://github.com/airspeed-velocity/asv
```
Now you should be ready to run `asv run` or whatever other command
diff --git a/python/asv-build.sh b/python/asv-build.sh
index 2bbc94b..99e6330 100755
--- a/python/asv-build.sh
+++ b/python/asv-build.sh
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -20,13 +20,17 @@
set -e
# ASV doesn't activate its conda environment for us
-source activate $ASV_ENV_PATH
+if [ -z "$ASV_ENV_DIR" ]; then exit 1; fi
+source activate $ASV_ENV_DIR
+echo "== Conda Prefix for benchmarks: " $CONDA_PREFIX " =="
# Build Arrow C++ libraries
export ARROW_BUILD_TOOLCHAIN=$CONDA_PREFIX
export ARROW_HOME=$CONDA_PREFIX
-
-echo $CONDA_PREFIX
+export PARQUET_HOME=$CONDA_PREFIX
+export ORC_HOME=$CONDA_PREFIX
+export PROTOBUF_HOME=$CONDA_PREFIX
+export BOOST_ROOT=$CONDA_PREFIX
pushd ../cpp
mkdir -p build
@@ -48,6 +52,7 @@ popd
# Build pyarrow wrappers
export SETUPTOOLS_SCM_PRETEND_VERSION=0.0.1
export PYARROW_BUILD_TYPE=release
+export PYARROW_PARALLEL=4
export PYARROW_WITH_PLASMA=1
python setup.py clean
diff --git a/python/asv-install.sh b/python/asv-install.sh
old mode 100644
new mode 100755
index 67d2c18..beef730
--- a/python/asv-install.sh
+++ b/python/asv-install.sh
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
diff --git a/python/asv-uninstall.sh b/python/asv-uninstall.sh
old mode 100644
new mode 100755
index 67d2c18..beef730
--- a/python/asv-uninstall.sh
+++ b/python/asv-uninstall.sh
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
diff --git a/python/asv.conf.json b/python/asv.conf.json
index 5f1d093..40938ee 100644
--- a/python/asv.conf.json
+++ b/python/asv.conf.json
@@ -35,9 +35,9 @@
// of the repository.
"repo_subdir": "python",
- "build_command": ["/bin/bash", "asv-build.sh"],
- "install_command": ["/bin/bash", "asv-install.sh"],
- "uninstall_command": ["/bin/bash", "asv-uninstall.sh"],
+ "build_command": ["/bin/bash {build_dir}/asv-build.sh"],
+ "install_command": ["/bin/bash {build_dir}/asv-install.sh"],
+ "uninstall_command": ["/bin/bash {build_dir}/asv-uninstall.sh"],
// List of branches to benchmark. If not provided, defaults to "master"
// (for git) or "default" (for mercurial).
@@ -81,11 +81,11 @@
// "pip+emcee": [""], // emcee is only available for install with pip.
// },
"matrix": {
- "boost-cpp": [],
+ "boost-cpp": ["1.67.0"],
"brotli": [],
"cmake": [],
"cython": [],
- "flatbuffers": [],
+ "flatbuffers": ["1.7.1"],
"libprotobuf": [],
"lz4-c": [],
"ninja": [],
diff --git a/python/benchmarks/parquet.py b/python/benchmarks/parquet.py
index 5623095..fd61793 100644
--- a/python/benchmarks/parquet.py
+++ b/python/benchmarks/parquet.py
@@ -21,7 +21,10 @@ import shutil
import tempfile
import pyarrow as pa
-import pyarrow.parquet as pq
+try:
+ import pyarrow.parquet as pq
+except ImportError:
+ pq = None
class ParquetManifestCreation(object):
@@ -34,6 +37,9 @@ class ParquetManifestCreation(object):
params = [(10, 100, 1000), (1, 8)]
def setup(self, num_partitions, num_threads):
+ if pq is None:
+ raise NotImplementedError
+
self.tmpdir = tempfile.mkdtemp('benchmark_parquet')
num1 = [random.choice(range(0, num_partitions))
for _ in range(self.size)]