You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by gr...@apache.org on 2020/11/27 05:40:18 UTC
[kudu] 04/04: WIP: Build on Apple M1 (arm64)
This is an automated email from the ASF dual-hosted git repository.
granthenke pushed a commit to branch apple-m1
in repository https://gitbox.apache.org/repos/asf/kudu.git
commit b0c8d2f59d69a005e69fb779817b6d80aa9b595e
Author: Grant Henke <gr...@gmail.com>
AuthorDate: Thu Nov 26 23:38:51 2020 -0600
WIP: Build on Apple M1 (arm64)
This is a WIP Patch to build Kudu on Apple M1 and
Big Sur (OSX 11).
Environment setup log:
# Setup passwordless sudo
sudo visudo
# Edit the line "%admin ALL=(ALL) ALL" to "%admin ALL=(ALL) NOPASSWD: ALL"
# Install xcode
sudo xcodebuild -license
sudo xcode-select --install
# TODO: Document to use xcode git (remove git from below list)
# Validate git is comming from xcode
# Example: git version 2.24.3 (Apple Git-128)
git --version
# Install homebrew
cd ~
mkdir homebrew && curl -L https://github.com/Homebrew/brew/tarball/master | tar xz --strip 1 -C homebrew
sudo mv homebrew /opt/homebrew
cd /opt/homebrew/bin
./brew update
./brew doctor # Fix any issues
cd ~
# Install Java
# Currently Azul has Arm 64-bit macOS releases:
# https://www.azul.com/downloads/zulu-community/?os=macos&architecture=arm-64-bit&package=jdk
curl https://cdn.azul.com/zulu/bin/zulu8.50.0.1013-ca-jdk8.0.275-macos_aarch64.zip -o zulu8.50.0.1013-ca-jdk8.0.275-macos_aarch64.zip
unzip zulu8.50.0.1013-ca-jdk8.0.275-macos_aarch64.zip
sudo mv zulu8.50.0.1013-ca-jdk8.0.275-macos_aarch64/zulu-8.jdk /Library/Java/JavaVirtualMachines/zulu-8.jdk
JAVA_HOME=`sudo /usr/libexec/java_home -v 8` java -version
# I recomend installing https://github.com/ohmyzsh/ohmyzsh
# Update your ~/.zshrc to include the following:
# Add the Homebrew bin to the PATH
export PATH="/opt/homebrew/bin:$PATH"
# Add the Homebrew OpenSSL to the environment
export PATH="/opt/homebrew/opt/openssl/bin:$PATH"
export LDFLAGS="-L/opt/homebrew/opt/openssl/lib"
export CPPFLAGS="-I/opt/homebrew/opt/openssl/include"
export PKG_CONFIG_PATH="/opt/homebrew/opt/openssl/lib/pkgconfig"
# Set JAVA_HOME
export JAVA_HOME=`sudo /usr/libexec/java_home -v 8`
# Reload your .zshrc
source ~/.zshrc
# TODO: Document the -s
# Install the required Kudu packages.
brew install -s autoconf automake cmake krb5 libtool openssl pkg-config pstree
# ccache doesn't work :(
brew install -s ccache
Build Log:
# Clone and build Kudu
mkdir ~/source
cd ~/source
git clone https://github.com/apache/kudu
cd kudu
thirdparty/build-if-necessary.sh
# TODO: Figure out why cmake cant find g++
# Added -DCMAKE_CXX_COMPILER=/usr/bin/g++ for now.
mkdir -p build/debug
cd build/debug
../../thirdparty/installed/common/bin/cmake \
-DCMAKE_BUILD_TYPE=debug \
-DOPENSSL_ROOT_DIR=/usr/local/opt/openssl \
-DCMAKE_CXX_COMPILER=/usr/bin/g++ \
../..make -
ctest
---
CMakeLists.txt | 34 +++++++++++++++++----------
java/gradle/protobuf.gradle | 5 +++-
src/kudu/gutil/CMakeLists.txt | 2 +-
src/kudu/gutil/atomicops-internals-macosx.h | 4 +++-
thirdparty/download-thirdparty.sh | 2 +-
thirdparty/patches/gperftools-osx-arm64.patch | 21 +++++++++++++++++
thirdparty/vars.sh | 2 +-
7 files changed, 52 insertions(+), 18 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 3350953..f88f4ba 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -157,6 +157,13 @@ include(CompilerInfo)
# compiler flags that are common across debug/release builds
execute_process(COMMAND uname -m OUTPUT_VARIABLE ARCH_NAME)
+message(STATUS "Found ARCH_NAME: ${ARCH_NAME}")
+# The Apple backend for aarch64 is called arm64.
+# Rename to aarch64 to simplify all of our build checks.
+if("${ARCH_NAME}" MATCHES "arm64")
+ set(ARCH_NAME "aarch64")
+endif()
+message(STATUS "Using ARCH_NAME: ${ARCH_NAME}")
if("${ARCH_NAME}" MATCHES "aarch64")
# Certain platforms such as ARM do not use signed chars by default
# which causes issues with certain bounds checks.
@@ -1084,19 +1091,20 @@ endif()
## Google PerfTools
##
## Disabled with TSAN/ASAN.
-if (NOT "${KUDU_USE_ASAN}" AND
- NOT "${KUDU_USE_TSAN}")
- find_package(GPerf REQUIRED)
- ADD_THIRDPARTY_LIB(tcmalloc
- STATIC_LIB "${TCMALLOC_STATIC_LIB}"
- SHARED_LIB "${TCMALLOC_SHARED_LIB}")
- ADD_THIRDPARTY_LIB(profiler
- STATIC_LIB "${PROFILER_STATIC_LIB}"
- SHARED_LIB "${PROFILER_SHARED_LIB}")
- list(APPEND KUDU_BASE_LIBS tcmalloc profiler)
- add_definitions("-DTCMALLOC_ENABLED")
- set(KUDU_TCMALLOC_AVAILABLE 1)
-endif()
+# TODO: Figure out why tcmalloc hangs protoc on OSX aarch64
+#if (NOT "${KUDU_USE_ASAN}" AND
+# NOT "${KUDU_USE_TSAN}")
+# find_package(GPerf REQUIRED)
+# ADD_THIRDPARTY_LIB(tcmalloc
+# STATIC_LIB "${TCMALLOC_STATIC_LIB}"
+# SHARED_LIB "${TCMALLOC_SHARED_LIB}")
+# ADD_THIRDPARTY_LIB(profiler
+# STATIC_LIB "${PROFILER_STATIC_LIB}"
+# SHARED_LIB "${PROFILER_SHARED_LIB}")
+# list(APPEND KUDU_BASE_LIBS tcmalloc profiler)
+# add_definitions("-DTCMALLOC_ENABLED")
+# set(KUDU_TCMALLOC_AVAILABLE 1)
+#endif()
## curl
find_package(CURL REQUIRED)
diff --git a/java/gradle/protobuf.gradle b/java/gradle/protobuf.gradle
index 5bcabd3..0ee4c45 100644
--- a/java/gradle/protobuf.gradle
+++ b/java/gradle/protobuf.gradle
@@ -22,7 +22,10 @@ apply plugin: "com.google.protobuf"
// Use the maven protoc artifact instead of local.
protobuf {
protoc {
- artifact = libs.protoc
+ // TODO: switch between artifacy and thirdparty protoc via command line property
+ // On Apple aarch64 use thirdparty by default.
+ path = "$rootDir/../thirdparty/installed/uninstrumented/bin/protoc"
+ // artifact = libs.protoc
}
}
diff --git a/src/kudu/gutil/CMakeLists.txt b/src/kudu/gutil/CMakeLists.txt
index 02808d9..460c975 100644
--- a/src/kudu/gutil/CMakeLists.txt
+++ b/src/kudu/gutil/CMakeLists.txt
@@ -50,7 +50,7 @@ set(GUTIL_SRCS
utf/rune.c
walltime.cc)
-if (NOT CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64")
+if (NOT "${ARCH_NAME}" MATCHES "aarch64")
set(GUTIL_SRCS
atomicops-internals-x86.cc
${GUTIL_SRCS})
diff --git a/src/kudu/gutil/atomicops-internals-macosx.h b/src/kudu/gutil/atomicops-internals-macosx.h
index a260d3b..c170957 100644
--- a/src/kudu/gutil/atomicops-internals-macosx.h
+++ b/src/kudu/gutil/atomicops-internals-macosx.h
@@ -274,7 +274,9 @@ inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) {
// are in a spinlock wait loop and should allow other hyperthreads
// to run, not speculate memory access, etc.
inline void PauseCPU() {
- __asm__ __volatile__("pause" : : : "memory");
+ // TODO: Only on OSX aarch64
+ __asm__ __volatile__("yield" : : : "memory");
+ // __asm__ __volatile__("pause" : : : "memory");
}
inline void Acquire_Store(volatile Atomic64 *ptr, Atomic64 value) {
diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh
index 2edb494..630aa7a 100755
--- a/thirdparty/download-thirdparty.sh
+++ b/thirdparty/download-thirdparty.sh
@@ -199,7 +199,7 @@ fetch_and_patch \
$GPERFTOOLS_SOURCE \
$GPERFTOOLS_PATCHLEVEL \
"patch -p1 < $TP_DIR/patches/gperftools-Replace-namespace-base-with-namespace-tcmalloc.patch" \
- "patch -p1 < $TP_DIR/patches/gperftools-unbreak-memz.patch" \
+ "patch -p1 < $TP_DIR/patches/gperftools-osx-arm64.patch" \
"autoreconf -fvi"
PROTOBUF_PATCHLEVEL=0
diff --git a/thirdparty/patches/gperftools-osx-arm64.patch b/thirdparty/patches/gperftools-osx-arm64.patch
new file mode 100644
index 0000000..7c4cdec
--- /dev/null
+++ b/thirdparty/patches/gperftools-osx-arm64.patch
@@ -0,0 +1,21 @@
+From 23f7072ab6bfa5c0d1b45cf1e8d66053c7b738a8 Mon Sep 17 00:00:00 2001
+From: Grant Henke <gr...@gmail.com>
+Date: Thu, 26 Nov 2020 20:56:46 -0600
+Subject: [PATCH] Add OS X arm64 program counter
+
+---
+ m4/pc_from_ucontext.m4 | 1 +
+ 1 file changed, 1 insertion(+)
+
+diff --git a/m4/pc_from_ucontext.m4 b/m4/pc_from_ucontext.m4
+index 0c7ee281..6e8b5d00 100644
+--- a/m4/pc_from_ucontext.m4
++++ b/m4/pc_from_ucontext.m4
+@@ -43,6 +43,7 @@ AC_DEFUN([AC_PC_FROM_UCONTEXT],
+ pc_fields="$pc_fields uc_mcontext->__ss.__rip" # OS X (>=10.5 [untested])
+ pc_fields="$pc_fields uc_mcontext->ss.srr0" # OS X (ppc, ppc64 [untested])
+ pc_fields="$pc_fields uc_mcontext->__ss.__srr0" # OS X (>=10.5 [untested])
++ pc_fields="$pc_fields uc_mcontext->__ss.__pc" # OS X (arm64 [untested])
+ pc_field_found=false
+ for pc_field in $pc_fields; do
+ if ! $pc_field_found; then
\ No newline at end of file
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index 25abe6a..cb74744 100644
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -50,7 +50,7 @@ GMOCK_VERSION=1.8.0
GMOCK_NAME=googletest-release-$GMOCK_VERSION
GMOCK_SOURCE=$TP_SOURCE_DIR/$GMOCK_NAME
-GPERFTOOLS_VERSION=2.6.90
+GPERFTOOLS_VERSION=2.8
GPERFTOOLS_NAME=gperftools-$GPERFTOOLS_VERSION
GPERFTOOLS_SOURCE=$TP_SOURCE_DIR/$GPERFTOOLS_NAME