You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ab...@apache.org on 2022/02/19 18:30:43 UTC
[tez] branch branch-0.9 updated: TEZ-4300: Download protoc automatically compile/development time (#115)
This is an automated email from the ASF dual-hosted git repository.
abstractdog pushed a commit to branch branch-0.9
in repository https://gitbox.apache.org/repos/asf/tez.git
The following commit(s) were added to refs/heads/branch-0.9 by this push:
new 23a855c TEZ-4300: Download protoc automatically compile/development time (#115)
23a855c is described below
commit 23a855c89921349188c4e12193d812a748f77c29
Author: Bodor Laszlo <bo...@gmail.com>
AuthorDate: Fri Feb 18 20:12:32 2022 +0100
TEZ-4300: Download protoc automatically compile/development time (#115)
---
BUILDING.txt | 6 +-
build-tools/.gitignore | 2 +
build-tools/install-protobuf.sh | 76 +++++++++++++++++++++++--
build-tools/{install-protobuf.sh => protocw} | 20 +++++--
pom.xml | 22 +++++++
tez-api/pom.xml | 22 +++++++
tez-plugins/tez-protobuf-history-plugin/pom.xml | 15 ++++-
7 files changed, 150 insertions(+), 13 deletions(-)
diff --git a/BUILDING.txt b/BUILDING.txt
index 875bf3e..ae81d9c 100644
--- a/BUILDING.txt
+++ b/BUILDING.txt
@@ -117,8 +117,10 @@ It's important to note that maven will still include tez-ui project, but all of
----------------------------------------------------------------------------------
Protocol Buffer compiler:
-The version of Protocol Buffer compiler, protoc, must be 2.5.0 and match the
-version of the protobuf JAR.
+The version of Protocol Buffer compiler, protoc, can be defined on-the-fly as:
+ $ mvn clean install -DskipTests -pl ./tez-api -Dprotobuf.version=3.7.1
+
+The default version is defined in the root pom.xml.
If you have multiple versions of protoc in your system, you can set in your
build shell the PROTOC_PATH environment variable to point to the one you
diff --git a/build-tools/.gitignore b/build-tools/.gitignore
new file mode 100644
index 0000000..adfc42e
--- /dev/null
+++ b/build-tools/.gitignore
@@ -0,0 +1,2 @@
+protobuf
+
diff --git a/build-tools/install-protobuf.sh b/build-tools/install-protobuf.sh
index 902049d..c28729a 100755
--- a/build-tools/install-protobuf.sh
+++ b/build-tools/install-protobuf.sh
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,7 +16,73 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-set -ex
-wget https://github.com/google/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz
-tar -xzvf protobuf-2.5.0.tar.gz
-cd protobuf-2.5.0 && ./configure --prefix=/usr && make && sudo make install
+# This script attempts to install an arbitrary version of protobuf if needed.
+# The desired version should be the first parameter: $1.
+# Typical usage: the script is automatically called from tez-api (by maven) during the build process.
+
+# This script runs from build-tools folder. The user can remove
+# the dynamically installed protobuf anytime like:
+# rm -rf ./build-tools/protobuf/ #from root folder
+
+set -x
+PROTOBUF_VERSION=${1:-2.5.0}
+PROTOBUF_MAJOR_VERSION=$(echo "$PROTOBUF_VERSION" | cut -d. -f1)
+if [ -n "$ZSH_VERSION" ]; then
+ SCRIPT_DIR="${0:a:h}"
+else
+ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+fi
+
+function install_protobuf {
+ # before protobuf 3, there is no pre-compiled executables are host on github, let's try to build and make it
+ if (( PROTOBUF_MAJOR_VERSION < 3 )); then
+ wget "https://github.com/google/protobuf/releases/download/v$PROTOBUF_VERSION/protobuf-$PROTOBUF_VERSION.tar.gz"
+ tar -xzvf "protobuf-$PROTOBUF_VERSION.tar.gz"
+ rm "protobuf-$PROTOBUF_VERSION.tar.gz"
+ cd "protobuf-$PROTOBUF_VERSION" && ./configure --prefix=/usr && make && sudo make install
+ # since protobuf 3, there are precompiled protoc executables on github, let's quickly download and use it
+ else
+ ARCH=`uname -m`
+ case "$(uname -s)" in
+ Darwin)
+ FILE_NAME="protoc-$PROTOBUF_VERSION-osx-$ARCH"
+ ;;
+ Linux)
+ if test $ARCH = "aarch64"; then
+ ARCH="aarch_64"
+ fi
+ FILE_NAME="protoc-$PROTOBUF_VERSION-linux-$ARCH"
+ ;;
+ *)
+ echo "Unsupported OS returned by uname -s, you'll have to install protobuf 3.x manually"
+ exit 1
+ ;;
+ esac
+ rm -f "$FILE_NAME.zip" #cleanup unfinished file if any
+ wget "https://github.com/google/protobuf/releases/download/v$PROTOBUF_VERSION/$FILE_NAME.zip"
+ mkdir "$SCRIPT_DIR/protobuf"
+ unzip -o "$FILE_NAME.zip" -d "$SCRIPT_DIR/protobuf"
+ rm "$FILE_NAME.zip"
+ fi
+}
+
+if test -f "$SCRIPT_DIR/protobuf/bin/protoc"; then
+ PROTOBUF_INSTALLED_VERSION=$("$SCRIPT_DIR/protobuf/bin/protoc" --version)
+else
+ PROTOBUF_INSTALLED_VERSION=$(protoc --version)
+fi
+
+PROTOC_EXIT_CODE=$?
+
+if [ $PROTOC_EXIT_CODE -eq 0 ]; then
+ PROTOBUF_INSTALLED_VERSION=$(echo "$PROTOBUF_INSTALLED_VERSION" | tr -s ' ' | cut -d ' ' -f 2)
+ if [ "$PROTOBUF_INSTALLED_VERSION" == "$PROTOBUF_VERSION" ]; then
+ echo "Current protobuf version is equal to the requested ($PROTOBUF_INSTALLED_VERSION), exiting..."
+ else
+ echo "Current protobuf version ($PROTOBUF_INSTALLED_VERSION) is not equal to the requested ($PROTOBUF_VERSION), installing $PROTOBUF_VERSION"
+ install_protobuf
+ fi
+else
+ echo "protoc --version command had non-zero return value, need to install probuf"
+ install_protobuf
+fi
diff --git a/build-tools/install-protobuf.sh b/build-tools/protocw
similarity index 67%
copy from build-tools/install-protobuf.sh
copy to build-tools/protocw
index 902049d..6196071 100755
--- a/build-tools/install-protobuf.sh
+++ b/build-tools/protocw
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,7 +16,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-set -ex
-wget https://github.com/google/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz
-tar -xzvf protobuf-2.5.0.tar.gz
-cd protobuf-2.5.0 && ./configure --prefix=/usr && make && sudo make install
+### This is a protoc wrapper for tez, which can dinamically call protoc from a downloaded protobuf.
+
+if [ -n "$ZSH_VERSION" ]; then
+ SCRIPT_DIR="${0:a:h}"
+else
+ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+fi
+
+if test -f "$SCRIPT_DIR/protobuf/bin/protoc"; then
+ "$SCRIPT_DIR/protobuf/bin/protoc" "$@"
+else
+ protoc "$@"
+fi
+exit $?
diff --git a/pom.xml b/pom.xml
index fc475fe..79d6a19 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1266,6 +1266,28 @@
</dependency>
</dependencies>
</profile>
+ <!-- This is because <protoc.path>${env.PROTOC_PATH}</protoc.path> above
+ doesn't let us define a default value in the absence of env.PROTOC_PATH.
+ By defining this profile, the following order is considered:
+
+ 0. protoc.path == env.PROTOC_PATH by pom.xml, if protoc.path is not defined
+ 1. -Dprotoc.path: if defined, it wins
+ 2. env.PROTOC_PATH: if protoc.path is not defined, but env.PROTOC_PATH is defined, env.PROTOC_PATH wins
+ (because protoc.path ==> env.PROTOC_PATH)
+ 3. if neither -Dprotoc.path, nor PROTOC_PATH is defined, protocw script will run
+ (which can run protoc from the PATH, or an automatically installed version from build-tools/protobuf)
+ -->
+ <profile>
+ <id>protoc-path-env-variable-not-defined</id>
+ <activation>
+ <property>
+ <name>!env.PROTOC_PATH</name>
+ </property>
+ </activation>
+ <properties>
+ <protoc.path>${basedir}/../build-tools/protocw</protoc.path>
+ </properties>
+ </profile>
</profiles>
<reporting>
diff --git a/tez-api/pom.xml b/tez-api/pom.xml
index 6b84a88..7d9c189 100644
--- a/tez-api/pom.xml
+++ b/tez-api/pom.xml
@@ -145,6 +145,28 @@
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
</plugin>
+ <!-- This plugin takes care of on-the-fly installation of the needed protobuf version.
+ The needed version is always what's defined as protobuf.version in the pom,
+ so if user wants to change protobuf version quickly in development time,
+ supposed to change only protobuf.version and then rebuild tez-api. -->
+ <plugin>
+ <artifactId>exec-maven-plugin</artifactId>
+ <groupId>org.codehaus.mojo</groupId>
+ <version>1.6.0</version>
+ <executions>
+ <execution>
+ <id>Install protobuf</id>
+ <phase>initialize</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <configuration>
+ <executable>${basedir}/../build-tools/install-protobuf.sh</executable>
+ <arguments>${protobuf.version}</arguments>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
<plugin>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-maven-plugins</artifactId>
diff --git a/tez-plugins/tez-protobuf-history-plugin/pom.xml b/tez-plugins/tez-protobuf-history-plugin/pom.xml
index 880aca9..8e74f89 100644
--- a/tez-plugins/tez-protobuf-history-plugin/pom.xml
+++ b/tez-plugins/tez-protobuf-history-plugin/pom.xml
@@ -85,8 +85,21 @@
</execution>
</executions>
</plugin>
-
</plugins>
</build>
+ <profiles>
+ <!-- For further details, please refer to the profile definition in root pom.xml -->
+ <profile>
+ <id>protoc-path-env-variable-not-defined</id>
+ <activation>
+ <property>
+ <name>!env.PROTOC_PATH</name>
+ </property>
+ </activation>
+ <properties>
+ <protoc.path>${basedir}/../../build-tools/protocw</protoc.path>
+ </properties>
+ </profile>
+ </profiles>
</project>