You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by fs...@apache.org on 2020/04/23 13:06:06 UTC
[arrow] branch master updated: ARROW-8497: [Archery] Add missing
components to build options
This is an automated email from the ASF dual-hosted git repository.
fsaintjacques pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 2409f4d ARROW-8497: [Archery] Add missing components to build options
2409f4d is described below
commit 2409f4d6445c7f331e758ecb8c7b510359cde757
Author: François Saint-Jacques <fs...@gmail.com>
AuthorDate: Thu Apr 23 09:05:42 2020 -0400
ARROW-8497: [Archery] Add missing components to build options
This is a cleanup to support to previous DefineOptions.cmake . It also adds a virtual component `--with-r` that enables the proper options for a R extension build.
Closes #6966 from fsaintjacques/ARROW-8497-archery-components
Authored-by: François Saint-Jacques <fs...@gmail.com>
Signed-off-by: François Saint-Jacques <fs...@gmail.com>
---
dev/archery/archery/cli.py | 78 ++++++++++++++++++++++--------
dev/archery/archery/lang/cpp.py | 104 +++++++++++++++++++++++++++++-----------
2 files changed, 133 insertions(+), 49 deletions(-)
diff --git a/dev/archery/archery/cli.py b/dev/archery/archery/cli.py
index c8af1b2..68c2742 100644
--- a/dev/archery/archery/cli.py
+++ b/dev/archery/archery/cli.py
@@ -134,36 +134,72 @@ def _apply_options(cmd, options):
help="CMake's CMAKE_BUILD_TYPE")
@click.option("--warn-level", default="production", type=warn_level_type,
help="Controls compiler warnings -W(no-)error.")
-# components
+@click.option("--use-gold-linker", default=True, type=BOOL,
+ help="Toggles ARROW_USE_LD_GOLD option.")
+# Tests and benchmarks
@click.option("--with-tests", default=True, type=BOOL,
help="Build with tests.")
-@click.option("--with-benchmarks", default=False, type=BOOL,
+@click.option("--with-benchmarks", default=None, type=BOOL,
help="Build with benchmarks.")
-@click.option("--with-examples", default=False, type=BOOL,
+@click.option("--with-examples", default=None, type=BOOL,
help="Build with examples.")
-@click.option("--with-python", default=False, type=BOOL,
- help="Build with python extension.")
-@click.option("--with-parquet", default=False, type=BOOL,
- help="Build with parquet file support.")
-@click.option("--with-gandiva", default=False, type=BOOL,
+@click.option("--with-integration", default=None, type=BOOL,
+ help="Build with integration test executables.")
+# Static checks
+@click.option("--use-asan", default=None, type=BOOL,
+ help="Toggle ARROW_USE_ASAN sanitizer.")
+@click.option("--use-tsan", default=None, type=BOOL,
+ help="Toggle ARROW_USE_TSAN sanitizer.")
+@click.option("--use-ubsan", default=None, type=BOOL,
+ help="Toggle ARROW_USE_UBSAN sanitizer.")
+@click.option("--with-fuzzing", default=None, type=BOOL,
+ help="Toggle ARROW_FUZZING.")
+# Components
+@click.option("--with-compute", default=None, type=BOOL,
+ help="Build the Arrow compute module.")
+@click.option("--with-csv", default=None, type=BOOL,
+ help="Build the Arrow CSV parser module.")
+@click.option("--with-cuda", default=None, type=BOOL,
+ help="Build the Arrow CUDA extensions.")
+@click.option("--with-dataset", default=None, type=BOOL,
+ help="Build the Arrow dataset module.")
+@click.option("--with-filesystem", default=None, type=BOOL,
+ help="Build the Arrow filesystem layer.")
+@click.option("--with-flight", default=None, type=BOOL,
+ help="Build with Flight rpc support.")
+@click.option("--with-gandiva", default=None, type=BOOL,
help="Build with Gandiva expression compiler support.")
-@click.option("--with-plasma", default=False, type=BOOL,
+@click.option("--with-hdfs", default=None, type=BOOL,
+ help="Build the Arrow HDFS bridge.")
+@click.option("--with-hiveserver2", default=None, type=BOOL,
+ help="Build the HiveServer2 client and arrow adapater.")
+@click.option("--with-ipc", default=None, type=BOOL,
+ help="Build the Arrow IPC extensions.")
+@click.option("--with-json", default=None, type=BOOL,
+ help="Build the Arrow JSON parser module.")
+@click.option("--with-jni", default=None, type=BOOL,
+ help="Build the Arrow JNI lib.")
+@click.option("--with-mimalloc", default=None, type=BOOL,
+ help="Build the Arrow mimalloc based allocator.")
+@click.option("--with-parquet", default=None, type=BOOL,
+ help="Build with Parquet file support.")
+@click.option("--with-plasma", default=None, type=BOOL,
help="Build with Plasma object store support.")
-@click.option("--with-flight", default=False, type=BOOL,
- help="Build with Flight rpc support.")
-@click.option("--with-compute", default=True, type=BOOL,
- help="Build with compute kernels support.")
-@click.option("--with-dataset", default=False, type=BOOL,
- help="Build with dataset support.")
-@click.option("--use-sanitizers", default=False, type=BOOL,
- help="Toggles ARROW_USE_*SAN sanitizers.")
-@click.option("--with-fuzzing", default=False, type=BOOL,
- help="Toggles ARROW_FUZZING.")
-@click.option("--use-gold-linker", default=True, type=BOOL,
- help="Toggles ARROW_USE_LD_GOLD option.")
+@click.option("--with-python", default=None, type=BOOL,
+ help="Build the Arrow CPython extesions.")
+@click.option("--with-r", default=None, type=BOOL,
+ help="Build the Arrow R extensions. This is not a CMake option, "
+ "it will toggle required options")
+@click.option("--with-s3", default=None, type=BOOL,
+ help="Build Arrow with S3 support.")
+# CMake extra feature
@click.option("--cmake-extras", type=str, multiple=True,
help="Extra flags/options to pass to cmake invocation. "
"Can be stacked")
+@click.option("--install-prefix", type=str,
+ help="Destination directory where files are installed. Expand to"
+ "CMAKE_INSTALL_PREFIX. Defaults to to $CONDA_PREFIX if the"
+ "variable exists.")
# misc
@click.option("-f", "--force", type=BOOL, is_flag=True, default=False,
help="Delete existing build directory if found.")
diff --git a/dev/archery/archery/lang/cpp.py b/dev/archery/archery/lang/cpp.py
index 07946ed..4bc3f10 100644
--- a/dev/archery/archery/lang/cpp.py
+++ b/dev/archery/archery/lang/cpp.py
@@ -39,17 +39,22 @@ class CppConfiguration:
build_type=None, warn_level=None,
cpp_package_prefix=None, install_prefix=None, use_conda=None,
# tests & examples
- with_tests=True, with_benchmarks=False, with_examples=False,
- # Languages support
- with_python=True,
- # Format support
- with_parquet=False,
+ with_tests=None, with_benchmarks=None, with_examples=None,
+ with_integration=None,
+ # static checks
+ use_asan=None, use_tsan=None, use_ubsan=None,
+ with_fuzzing=None,
# Components
- with_gandiva=False, with_compute=False, with_dataset=False,
- with_plasma=False, with_flight=False,
+ with_compute=None, with_csv=None, with_cuda=None,
+ with_dataset=None, with_filesystem=None, with_flight=None,
+ with_gandiva=None, with_hdfs=None, with_hiveserver2=None,
+ with_ipc=None, with_json=None, with_jni=None,
+ with_mimalloc=None,
+ with_parquet=None, with_plasma=None, with_python=True,
+ with_r=None, with_s3=None,
# extras
- with_lint_only=False, with_fuzzing=False,
- use_gold_linker=True, use_sanitizers=True,
+ with_lint_only=False,
+ use_gold_linker=True,
cmake_extras=None):
self._cc = cc
self._cxx = cxx
@@ -64,21 +69,50 @@ class CppConfiguration:
self.with_tests = with_tests
self.with_benchmarks = with_benchmarks
self.with_examples = with_examples
- self.with_python = with_python
- self.with_parquet = with_parquet or with_dataset
- self.with_gandiva = with_gandiva
- self.with_plasma = with_plasma
- self.with_flight = with_flight
+ self.with_integration = with_integration
+
+ self.use_asan = use_asan
+ self.use_tsan = use_tsan
+ self.use_ubsan = use_ubsan
+ self.with_fuzzing = with_fuzzing
+
self.with_compute = with_compute
+ self.with_csv = with_csv
+ self.with_cuda = with_cuda
self.with_dataset = with_dataset
+ self.with_filesystem = with_filesystem
+ self.with_flight = with_flight
+ self.with_gandiva = with_gandiva
+ self.with_hdfs = with_hdfs
+ self.with_hiveserver2 = with_hiveserver2
+ self.with_ipc = with_ipc
+ self.with_json = with_json
+ self.with_jni = with_jni
+ self.with_mimalloc = with_mimalloc
+ self.with_parquet = with_parquet
+ self.with_plasma = with_plasma
+ self.with_python = with_python
+ self.with_r = with_r
+ self.with_s3 = with_s3
self.with_lint_only = with_lint_only
- self.with_fuzzing = with_fuzzing
self.use_gold_linker = use_gold_linker
- self.use_sanitizers = use_sanitizers
self.cmake_extras = cmake_extras
+ # Fixup required dependencies
+ if self.with_r:
+ self.with_csv = True
+ self.with_dataset = True
+ self.with_filesystem = True
+ self.with_ipc = True
+ self.with_json = True
+ self.with_parquet = True
+
+ if self.with_dataset:
+ self.with_filesystem = True
+ self.with_parquet = True
+
@property
def build_type(self):
if self._build_type:
@@ -131,31 +165,45 @@ class CppConfiguration:
yield ("ARROW_DEPENDENCY_SOURCE", "SYSTEM")
yield ("ARROW_PACKAGE_PREFIX", self._package_prefix)
+ # Tests and benchmarks
yield ("ARROW_BUILD_TESTS", truthifier(self.with_tests))
yield ("ARROW_BUILD_BENCHMARKS", truthifier(self.with_benchmarks))
yield ("ARROW_BUILD_EXAMPLES", truthifier(self.with_examples))
+ yield ("ARROW_BUILD_INTEGRATION", truthifier(self.with_integration))
- yield ("ARROW_PYTHON", truthifier(self.with_python))
+ # Static checks
+ yield ("ARROW_USE_ASAN", truthifier(self.use_asan))
+ yield ("ARROW_USE_TSAN", truthifier(self.use_tsan))
+ yield ("ARROW_USE_UBSAN", truthifier(self.use_ubsan))
+ yield ("ARROW_FUZZING", truthifier(self.with_fuzzing))
+ # Components
+ yield ("ARROW_COMPUTE", truthifier(self.with_compute))
+ yield ("ARROW_CSV", truthifier(self.with_csv))
+ yield ("ARROW_CUDA", truthifier(self.with_cuda))
+ yield ("ARROW_DATASET", truthifier(self.with_dataset))
+ yield ("ARROW_FILESYSTEM", truthifier(self.with_filesystem))
+ yield ("ARROW_FLIGHT", truthifier(self.with_flight))
+ yield ("ARROW_GANDIVA", truthifier(self.with_gandiva))
if self.with_parquet:
yield ("ARROW_PARQUET", truthifier(self.with_parquet))
yield ("ARROW_WITH_BROTLI", "ON")
yield ("ARROW_WITH_SNAPPY", "ON")
-
- yield ("ARROW_GANDIVA", truthifier(self.with_gandiva))
+ yield ("ARROW_HDFS", truthifier(self.with_hdfs))
+ yield ("ARROW_HIVESERVER2", truthifier(self.with_hiveserver2))
+ yield ("ARROW_IPC", truthifier(self.with_ipc))
+ yield ("ARROW_JSON", truthifier(self.with_json))
+ yield ("ARROW_JNI", truthifier(self.with_jni))
+ yield ("ARROW_MIMALLOC", truthifier(self.with_mimalloc))
yield ("ARROW_PLASMA", truthifier(self.with_plasma))
- yield ("ARROW_FLIGHT", truthifier(self.with_flight))
- yield ("ARROW_COMPUTE", truthifier(self.with_compute))
- yield ("ARROW_DATASET", truthifier(self.with_dataset))
-
- if self.use_sanitizers or self.with_fuzzing:
- yield ("ARROW_USE_ASAN", "ON")
- yield ("ARROW_USE_UBSAN", "ON")
+ yield ("ARROW_PYTHON", truthifier(self.with_python))
+ yield ("ARROW_S3", truthifier(self.with_s3))
yield ("ARROW_LINT_ONLY", truthifier(self.with_lint_only))
- yield ("ARROW_FUZZING", truthifier(self.with_fuzzing))
- if self.use_gold_linker and not self.with_fuzzing:
+ # Some configurations don't like gnu gold linker.
+ broken_with_gold_ld = [self.with_fuzzing, self.with_gandiva]
+ if self.use_gold_linker and not any(broken_with_gold_ld):
yield ("ARROW_USE_LD_GOLD", truthifier(self.use_gold_linker))
# Detect custom conda toolchain