You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by uw...@apache.org on 2017/07/10 17:32:50 UTC
arrow git commit: ARROW-1196: [C++] Release, Debug, Toolchain, NMake Generator Appveyor…
Repository: arrow
Updated Branches:
refs/heads/master cab07c2bd -> bc16e0e04
ARROW-1196: [C++] Release, Debug, Toolchain, NMake Generator Appveyor…
… jobs; Windows.md updated.
Author: Max Risuhin <ri...@gmail.com>
Closes #823 from MaxRis/ARROW-1196 and squashes the following commits:
1ca1155 [Max Risuhin] ARROW-1196: [C++] Release, Debug, Toolchain, NMake Generator Appveyor jobs; Windows.md updated.
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/bc16e0e0
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/bc16e0e0
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/bc16e0e0
Branch: refs/heads/master
Commit: bc16e0e04a3d4ea9c96579ac59cee4b16e771850
Parents: cab07c2
Author: Max Risuhin <ri...@gmail.com>
Authored: Mon Jul 10 19:32:46 2017 +0200
Committer: Uwe L. Korn <uw...@xhochy.com>
Committed: Mon Jul 10 19:32:46 2017 +0200
----------------------------------------------------------------------
appveyor.yml | 15 +++
ci/msvc-build.bat | 31 ++++-
cpp/CMakeLists.txt | 4 +
cpp/apidoc/Windows.md | 146 ++++++++++++++++++++---
cpp/cmake_modules/ThirdpartyToolchain.cmake | 7 +-
5 files changed, 181 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/bc16e0e0/appveyor.yml
----------------------------------------------------------------------
diff --git a/appveyor.yml b/appveyor.yml
index 802d43b..f3ce5a5 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -20,9 +20,23 @@ os: Visual Studio 2015
environment:
matrix:
+ - GENERATOR: NMake Makefiles
+ PYTHON: "3.5"
+ ARCH: "64"
+ CONFIGURATION: "Release"
+ - GENERATOR: Visual Studio 14 2015 Win64
+ PYTHON: "3.5"
+ ARCH: "64"
+ CONFIGURATION: "Debug"
- GENERATOR: Visual Studio 14 2015 Win64
PYTHON: "3.5"
ARCH: "64"
+ CONFIGURATION: "Release"
+ - GENERATOR: Visual Studio 14 2015 Win64
+ PYTHON: "3.5"
+ ARCH: "64"
+ CONFIGURATION: "Toolchain"
+
MSVC_DEFAULT_OPTIONS: ON
BOOST_ROOT: C:\Libraries\boost_1_63_0
BOOST_LIBRARYDIR: C:\Libraries\boost_1_63_0\lib64-msvc-14.0
@@ -30,6 +44,7 @@ environment:
init:
- set MINICONDA=C:\Miniconda35-x64
- set PATH=%MINICONDA%;%MINICONDA%/Scripts;%MINICONDA%/Library/bin;%PATH%
+ - if "%GENERATOR%"=="NMake Makefiles" call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x86_amd64
build_script:
- git config core.symlinks true
http://git-wip-us.apache.org/repos/asf/arrow/blob/bc16e0e0/ci/msvc-build.bat
----------------------------------------------------------------------
diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat
index 366fc28..c84d12d 100644
--- a/ci/msvc-build.bat
+++ b/ci/msvc-build.bat
@@ -17,18 +17,41 @@
@echo on
+if "%CONFIGURATION%" == "Debug" (
+ mkdir cpp\build-debug
+ pushd cpp\build-debug
+
+ cmake -G "%GENERATOR%" ^
+ -DARROW_BOOST_USE_SHARED=OFF ^
+ -DCMAKE_BUILD_TYPE=Debug ^
+ -DARROW_CXXFLAGS="/MP" ^
+ .. || exit /B
+
+ cmake --build . --config Debug || exit /B
+ popd
+
+ @rem Finish Debug build successfully
+ exit /B 0
+)
+
conda update --yes --quiet conda
conda create -n arrow -q -y python=%PYTHON% ^
six pytest setuptools numpy pandas cython
-conda install -n arrow -q -y -c conda-forge ^
- flatbuffers rapidjson ^
- cmake git boost-cpp thrift-cpp snappy zlib brotli gflags
+
+if "%CONFIGURATION%" == "Toolchain" (
+ conda install -n arrow -q -y -c conda-forge ^
+ flatbuffers rapidjson cmake git boost-cpp ^
+ thrift-cpp snappy zlib brotli gflags lz4-c zstd
+)
call activate arrow
+if "%CONFIGURATION%" == "Toolchain" (
+ set ARROW_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library
+)
+
set ARROW_HOME=%CONDA_PREFIX%\Library
-set ARROW_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library
@rem Build and test Arrow C++ libraries
http://git-wip-us.apache.org/repos/asf/arrow/blob/bc16e0e0/cpp/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 6d01fd9..002a07e 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -152,6 +152,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
"Snappy static lib suffix used on Windows with MSVC (default is empty string)")
set(ZLIB_MSVC_STATIC_LIB_SUFFIX "libstatic" CACHE STRING
"Zlib static lib suffix used on Windows with MSVC (default libstatic)")
+ set(LZ4_MSVC_STATIC_LIB_SUFFIX "_static" CACHE STRING
+ "Lz4 static lib suffix used on Windows with MSVC (default _static)")
+ set(ZSTD_MSVC_STATIC_LIB_SUFFIX "_static" CACHE STRING
+ "ZStd static lib suffix used on Windows with MSVC (default _static)")
endif()
endif()
http://git-wip-us.apache.org/repos/asf/arrow/blob/bc16e0e0/cpp/apidoc/Windows.md
----------------------------------------------------------------------
diff --git a/cpp/apidoc/Windows.md b/cpp/apidoc/Windows.md
index b9a6143..6bfb951 100644
--- a/cpp/apidoc/Windows.md
+++ b/cpp/apidoc/Windows.md
@@ -21,24 +21,13 @@ investing time in maintaining the thirdparty build dependencies for Arrow and
related C++ libraries using the conda package manager. Others are free to add
other development instructions for Windows here.
-### Visual Studio
-
-Microsoft provides the free Visual Studio 2017 Community edition. When doing
-development, you must launch the developer command prompt using
-
-```
-"C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\Tools\VsDevCmd.bat" -arch=amd64
-```
-
-It's easiest to configure a console emulator like [cmder][3] to automatically
-launch this when starting a new development console.
-
### conda and package toolchain
[Miniconda][1] is a minimal Python distribution including the conda package
manager. To get started, download and install a 64-bit distribution.
-We recommend using packages from [conda-forge][2]
+We recommend using packages from [conda-forge][2].
+Launch cmd.exe and run following commands:
```shell
conda config --add channels conda-forge
@@ -47,9 +36,92 @@ conda config --add channels conda-forge
Now, you can bootstrap a build environment
```shell
-conda create -n arrow-dev cmake git boost
+conda create -n arrow-dev cmake git boost-cpp flatbuffers rapidjson cmake thrift-cpp snappy zlib brotli gflags lz4-c zstd
```
+Activate just created conda environment with pre-installed packages from
+previous step:
+
+```shell
+activate arrow-dev
+```
+
+We are using [cmake][4] tool to support Windows builds.
+To allow cmake to pick up 3rd party dependencies, you should set
+`ARROW_BUILD_TOOLCHAIN` environment variable to contain `Library` folder
+path of new created on previous step `arrow-dev` conda environment.
+For instance, if `Miniconda` was installed to default destination, `Library`
+folder path for `arrow-dev` conda environment will be as following:
+
+```shell
+C:\Users\YOUR_USER_NAME\Miniconda3\envs\arrow-dev\Library
+```
+
+To set `ARROW_BUILD_TOOLCHAIN` environment variable visible only for current terminal session you can run following:
+```shell
+set ARROW_BUILD_TOOLCHAIN=C:\Users\YOUR_USER_NAME\Miniconda3\envs\arrow-dev\Library
+```
+
+To validate value of `ARROW_BUILD_TOOLCHAIN` environment variable you can run following terminal command:
+```shell
+echo %ARROW_BUILD_TOOLCHAIN%
+```
+
+As alternative to `ARROW_BUILD_TOOLCHAIN`, it's possible to configure path
+to each 3rd party dependency separately by setting appropriate environment
+variable:
+
+`FLATBUFFERS_HOME` variable with path to `flatbuffers` installation
+`RAPIDJSON_HOME` variable with path to `rapidjson` installation
+`GFLAGS_HOME` variable with path to `gflags` installation
+`SNAPPY_HOME` variable with path to `snappy` installation
+`ZLIB_HOME` variable with path to `zlib` installation
+`BROTLI_HOME` variable with path to `brotli` installation
+`LZ4_HOME` variable with path to `lz4` installation
+`ZSTD_HOME` variable with path to `zstd` installation
+
+### Customize static libraries names lookup of 3rd party dependencies
+
+If you decided to use pre-built 3rd party dependencies libs, it's possible to
+configure Arrow's cmake build script to search for customized names of 3rd
+party static libs.
+
+`zlib`. Pass `-DARROW_ZLIB_VENDORED=OFF` to enable lookup of custom zlib
+build. Set `ZLIB_HOME` environment variable. Pass
+`-DZLIB_MSVC_STATIC_LIB_SUFFIX=%ZLIB_SUFFIX%` to link with z%ZLIB_SUFFIX%.lib
+
+`brotli`. Set `BROTLY_HOME` environment variable. Pass
+`-DBROTLI_MSVC_STATIC_LIB_SUFFIX=%BROTLI_SUFFIX%` to link with
+brotli*%BROTLI_SUFFIX%.lib.
+
+`snappy`. Set `SNAPPY_HOME` environment variable. Pass
+`-DSNAPPY_MSVC_STATIC_LIB_SUFFIX=%SNAPPY_SUFFIX%` to link with
+snappy%SNAPPY_SUFFIX%.lib.
+
+`lz4`. Set `LZ4_HOME` environment variable. Pass
+`-LZ4_MSVC_STATIC_LIB_SUFFIX=%LZ4_SUFFIX%` to link with
+lz4%LZ4_SUFFIX%.lib.
+
+`zstd`. Set `ZSTD_HOME` environment variable. Pass
+`-ZSTD_MSVC_STATIC_LIB_SUFFIX=%ZSTD_SUFFIX%` to link with
+zstd%ZSTD_SUFFIX%.lib.
+
+### Visual Studio
+
+Microsoft provides the free Visual Studio Community edition. When doing
+development, you must launch the developer command prompt using
+
+#### Visual Studio 2015
+
+```"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" amd64```
+
+#### Visual Studio 2017
+
+```"C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\Tools\VsDevCmd.bat" -arch=amd64```
+
+It's easiest to configure a console emulator like [cmder][3] to automatically
+launch this when starting a new development console.
+
## Building with NMake
Activate your conda build environment:
@@ -58,7 +130,8 @@ Activate your conda build environment:
activate arrow-dev
```
-Now, do an out of source build using `nmake`:
+Change working directory in cmd.exe to the root directory of Arrow and
+do an out of source build using `nmake`:
```
cd cpp
@@ -72,14 +145,53 @@ When using conda, only release builds are currently supported.
## Build using Visual Studio (MSVC) Solution Files
-To build on the command line by instead generating a MSVC solution, instead
-run:
+Activate your conda build environment:
```
+activate arrow-dev
+```
+
+Change working directory in cmd.exe to the root directory of Arrow and
+do an out of source build by generating a MSVC solution:
+
+```
+cd cpp
+mkdir build
+cd build
cmake -G "Visual Studio 14 2015 Win64" -DCMAKE_BUILD_TYPE=Release ..
cmake --build . --config Release
```
+## Debug build
+
+To build Debug version of Arrow you should have pre-insalled Debug version of
+boost libs.
+
+It's recommended to configure cmake build with following variables for Debug build:
+
+`-DARROW_BOOST_USE_SHARED=OFF` - enables static linking with boost debug libs and
+simplifies run-time loading of 3rd parties. (Recommended)
+
+`-DBOOST_ROOT` - sets the root directory of boost libs. (Optional)
+
+`-DBOOST_LIBRARYDIR` - sets the directory with boost lib files. (Optional)
+
+Command line to build Arrow in Debug might look as following:
+
+```
+cd cpp
+mkdir build
+cd build
+cmake -G "Visual Studio 14 2015 Win64" -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=OFF -DCMAKE_BUILD_TYPE=Debug -DBOOST_ROOT=C:/local/boost_1_63_0 -DBOOST_LIBRARYDIR=C:/local/boost_1_63_0/lib64-msvc-14.0 ..
+cmake --build . --config Debug
+```
+
+To get the latest build instructions, you can reference [msvc-build.bat][5], which is used by automated Appveyor builds.
+
+
+
[1]: https://conda.io/miniconda.html
[2]: https://conda-forge.github.io/
[3]: http://cmder.net/
+[4]: https://cmake.org/
+[5]: https://github.com/apache/arrow/blob/master/ci/msvc-build.bat
http://git-wip-us.apache.org/repos/asf/arrow/blob/bc16e0e0/cpp/cmake_modules/ThirdpartyToolchain.cmake
----------------------------------------------------------------------
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index 3eef2f7..74939ac 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -346,10 +346,15 @@ if (ARROW_IPC)
## Flatbuffers
if("${FLATBUFFERS_HOME}" STREQUAL "")
set(FLATBUFFERS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/flatbuffers_ep-prefix/src/flatbuffers_ep-install")
+ if (MSVC)
+ set(FLATBUFFERS_CMAKE_CXX_FLAGS /EHsc)
+ else()
+ set(FLATBUFFERS_CMAKE_CXX_FLAGS -fPIC)
+ endif()
ExternalProject_Add(flatbuffers_ep
URL "https://github.com/google/flatbuffers/archive/v${FLATBUFFERS_VERSION}.tar.gz"
CMAKE_ARGS
- "-DCMAKE_CXX_FLAGS=-fPIC"
+ "-DCMAKE_CXX_FLAGS=${FLATBUFFERS_CMAKE_CXX_FLAGS}"
"-DCMAKE_INSTALL_PREFIX:PATH=${FLATBUFFERS_PREFIX}"
"-DFLATBUFFERS_BUILD_TESTS=OFF")