You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@orc.apache.org by om...@apache.org on 2019/10/23 17:49:23 UTC
[orc] branch branch-1.5 updated (8d85c8f -> 61e9d00)
This is an automated email from the ASF dual-hosted git repository.
omalley pushed a change to branch branch-1.5
in repository https://gitbox.apache.org/repos/asf/orc.git.
from 8d85c8f ORC-480: [C++] Add an option to succeed despite warnings
new 2ef409f ORC-560: Improve docker scripts and add centos 8 and debian 10.
new 61e9d00 ORC-552: Fix c++ compilation for centos 6 and 8, and MacOs 10.14.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
c++/src/ColumnPrinter.cc | 56 ++++++++++++++++----------------
c++/src/Vector.cc | 28 ++++++++--------
c++/test/TestColumnReader.cc | 3 ++
cmake_modules/FindSnappy.cmake | 2 +-
cmake_modules/ThirdpartyToolchain.cmake | 2 +-
docker/{centos7 => centos8}/Dockerfile | 12 ++-----
docker/{ubuntu16 => debian10}/Dockerfile | 12 +++----
docker/os-list.txt | 3 ++
docker/run-all.sh | 31 +++++++++---------
docker/{run-all.sh => run-one.sh} | 48 ++++++++++++---------------
10 files changed, 95 insertions(+), 102 deletions(-)
copy docker/{centos7 => centos8}/Dockerfile (83%)
copy docker/{ubuntu16 => debian10}/Dockerfile (89%)
copy docker/{run-all.sh => run-one.sh} (63%)
[orc] 01/02: ORC-560: Improve docker scripts and add centos 8 and
debian 10.
Posted by om...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
omalley pushed a commit to branch branch-1.5
in repository https://gitbox.apache.org/repos/asf/orc.git
commit 2ef409f4cd46af71e3018c60e093a9eaefd7cdfb
Author: Owen O'Malley <om...@apache.org>
AuthorDate: Thu Oct 17 06:57:54 2019 -0700
ORC-560: Improve docker scripts and add centos 8 and debian 10.
Fixes #437
Signed-off-by: Owen O'Malley <om...@apache.org>
---
docker/centos8/Dockerfile | 53 +++++++++++++++++++++++++++++++++++++++
docker/debian10/Dockerfile | 43 +++++++++++++++++++++++++++++++
docker/os-list.txt | 3 +++
docker/run-all.sh | 31 ++++++++++++-----------
docker/{run-all.sh => run-one.sh} | 48 ++++++++++++++++-------------------
5 files changed, 136 insertions(+), 42 deletions(-)
diff --git a/docker/centos8/Dockerfile b/docker/centos8/Dockerfile
new file mode 100644
index 0000000..4a6d2c6
--- /dev/null
+++ b/docker/centos8/Dockerfile
@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# ORC compile for CentOS 8
+#
+
+FROM centos:8
+MAINTAINER Owen O'Malley <ow...@hortonworks.com>
+
+RUN yum check-update || true
+RUN yum install -y \
+ cmake \
+ curl-devel \
+ cyrus-sasl-devel \
+ expat-devel \
+ gcc \
+ gcc-c++ \
+ gettext-devel \
+ git \
+ java-1.8.0-openjdk \
+ java-1.8.0-openjdk-devel \
+ libtool \
+ make \
+ maven \
+ openssl-devel \
+ tar \
+ wget \
+ which \
+ zlib-devel
+
+ENV TZ=America/Los_Angeles
+WORKDIR /root
+
+VOLUME /root/.m2/repository
+
+CMD git clone https://github.com/apache/orc.git -b master && \
+ mkdir orc/build && \
+ cd orc/build && \
+ cmake .. && \
+ make package test-out
diff --git a/docker/debian10/Dockerfile b/docker/debian10/Dockerfile
new file mode 100644
index 0000000..fb73d5b
--- /dev/null
+++ b/docker/debian10/Dockerfile
@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# ORC compile for Debian 10
+#
+
+FROM debian:buster
+MAINTAINER Owen O'Malley <om...@apache.org>
+
+RUN apt-get update
+RUN apt-get install -y \
+ cmake \
+ gcc \
+ g++ \
+ git \
+ libsasl2-dev \
+ libssl-dev \
+ openjdk-11-jdk \
+ make \
+ maven
+
+WORKDIR /root
+
+VOLUME /root/.m2/repository
+
+CMD git clone https://github.com/apache/orc.git -b master && \
+ mkdir orc/build && \
+ cd orc/build && \
+ cmake .. && \
+ make package test-out
diff --git a/docker/os-list.txt b/docker/os-list.txt
index dcb50f0..a0bf99d 100644
--- a/docker/os-list.txt
+++ b/docker/os-list.txt
@@ -1,6 +1,9 @@
centos6
centos7
+centos8
debian8
+debian9
+debian10
ubuntu12
ubuntu14
ubuntu16
diff --git a/docker/run-all.sh b/docker/run-all.sh
index d770bae..8f1436b 100755
--- a/docker/run-all.sh
+++ b/docker/run-all.sh
@@ -24,31 +24,32 @@ MAKEDIR="mkdir orc/build && cd orc/build"
VOLUME="--volume m2cache:/root/.m2/repository"
mkdir -p logs
+function failure {
+ echo "Failed tests"
+ grep -h "FAILED " logs/*-test.log
+ exit 1
+}
+
start=`date`
for os in `cat os-list.txt`; do
echo "Building $os"
( cd $os && docker build -t "orc-$os" . ) > logs/$os-build.log 2>&1 || exit 1
done
testStart=`date`
+rm -f logs/pids.txt
for os in `cat os-list.txt`; do
- echo "Testing $os"
- case $os in
- centos6|ubuntu12)
- OPTS="-DSNAPPY_HOME=/usr/local -DPROTOBUF_HOME=/usr/local"
- ;;
- centos7|debian8|ubuntu14)
- OPTS="-DSNAPPY_HOME=/usr/local"
- ;;
- *)
- OPTS=""
- ;;
- esac
- docker run $VOLUME "orc-$os" /bin/bash -c \
- "$CLONE && $MAKEDIR && cmake $OPTS .. && make package test-out" \
- > logs/$os-test.log 2>&1 || exit 1
+ ./run-one.sh $1 $2 $os > logs/$os-test.log 2>&1 &
+ echo "$!" >> logs/pids.txt
+ echo "Launching $os as $!"
+done
+
+for job in `cat logs/pids.txt`; do
+ echo "Waiting for $job"
+ wait $job || failure
done
+echo ""
echo "Build start: $start"
echo "Test start: $testStart"
echo "End:" `date`
diff --git a/docker/run-all.sh b/docker/run-one.sh
similarity index 63%
copy from docker/run-all.sh
copy to docker/run-one.sh
index d770bae..7688560 100755
--- a/docker/run-all.sh
+++ b/docker/run-one.sh
@@ -18,37 +18,31 @@
GITHUB_USER=$1
URL=https://github.com/$GITHUB_USER/orc.git
BRANCH=$2
+OS=$3
+
+function failure {
+ echo "FAILED $OS"
+ exit 1
+}
CLONE="git clone $URL -b $BRANCH"
MAKEDIR="mkdir orc/build && cd orc/build"
VOLUME="--volume m2cache:/root/.m2/repository"
-mkdir -p logs
-start=`date`
-for os in `cat os-list.txt`; do
- echo "Building $os"
- ( cd $os && docker build -t "orc-$os" . ) > logs/$os-build.log 2>&1 || exit 1
-done
-testStart=`date`
+echo "Started $GITHUB_USER/$BRANCH on $OS at $(date)"
-for os in `cat os-list.txt`; do
- echo "Testing $os"
- case $os in
- centos6|ubuntu12)
- OPTS="-DSNAPPY_HOME=/usr/local -DPROTOBUF_HOME=/usr/local"
- ;;
- centos7|debian8|ubuntu14)
- OPTS="-DSNAPPY_HOME=/usr/local"
- ;;
- *)
- OPTS=""
- ;;
- esac
- docker run $VOLUME "orc-$os" /bin/bash -c \
+case $OS in
+centos6|ubuntu12)
+ OPTS="-DSNAPPY_HOME=/usr/local -DPROTOBUF_HOME=/usr/local"
+ ;;
+centos7|debian8|ubuntu14)
+ OPTS="-DSNAPPY_HOME=/usr/local"
+ ;;
+*)
+ OPTS=""
+ ;;
+esac
+docker run $VOLUME "orc-$OS" /bin/bash -c \
"$CLONE && $MAKEDIR && cmake $OPTS .. && make package test-out" \
- > logs/$os-test.log 2>&1 || exit 1
-done
-
-echo "Build start: $start"
-echo "Test start: $testStart"
-echo "End:" `date`
+ || failure
+echo "Finished $OS at $(date)"
[orc] 02/02: ORC-552: Fix c++ compilation for centos 6 and 8,
and MacOs 10.14.
Posted by om...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
omalley pushed a commit to branch branch-1.5
in repository https://gitbox.apache.org/repos/asf/orc.git
commit 61e9d008d06a4f9291ee62737fbfd0e34f9b20d5
Author: Owen O'Malley <om...@apache.org>
AuthorDate: Mon Oct 14 14:40:06 2019 -0700
ORC-552: Fix c++ compilation for centos 6 and 8, and MacOs 10.14.
Fixes #436
Signed-off-by: Owen O'Malley <om...@apache.org>
---
c++/src/ColumnPrinter.cc | 56 ++++++++++++++++-----------------
c++/src/Vector.cc | 28 ++++++++---------
c++/test/TestColumnReader.cc | 3 ++
cmake_modules/FindSnappy.cmake | 2 +-
cmake_modules/ThirdpartyToolchain.cmake | 2 +-
5 files changed, 47 insertions(+), 44 deletions(-)
diff --git a/c++/src/ColumnPrinter.cc b/c++/src/ColumnPrinter.cc
index 379ef0b..b4b5860 100644
--- a/c++/src/ColumnPrinter.cc
+++ b/c++/src/ColumnPrinter.cc
@@ -289,8 +289,8 @@ namespace orc {
return std::unique_ptr<ColumnPrinter>(result);
}
- VoidColumnPrinter::VoidColumnPrinter(std::string& buffer
- ): ColumnPrinter(buffer) {
+ VoidColumnPrinter::VoidColumnPrinter(std::string& _buffer
+ ): ColumnPrinter(_buffer) {
// PASS
}
@@ -302,8 +302,8 @@ namespace orc {
writeString(buffer, "null");
}
- LongColumnPrinter::LongColumnPrinter(std::string& buffer
- ): ColumnPrinter(buffer),
+ LongColumnPrinter::LongColumnPrinter(std::string& _buffer
+ ): ColumnPrinter(_buffer),
data(nullptr) {
// PASS
}
@@ -324,9 +324,9 @@ namespace orc {
}
}
- DoubleColumnPrinter::DoubleColumnPrinter(std::string& buffer,
+ DoubleColumnPrinter::DoubleColumnPrinter(std::string& _buffer,
const Type& type
- ): ColumnPrinter(buffer),
+ ): ColumnPrinter(_buffer),
data(nullptr),
isFloat(type.getKind() == FLOAT){
// PASS
@@ -348,8 +348,8 @@ namespace orc {
}
}
- Decimal64ColumnPrinter::Decimal64ColumnPrinter(std::string& buffer
- ): ColumnPrinter(buffer),
+ Decimal64ColumnPrinter::Decimal64ColumnPrinter(std::string& _buffer
+ ): ColumnPrinter(_buffer),
data(nullptr),
scale(0) {
// PASS
@@ -398,8 +398,8 @@ namespace orc {
}
}
- Decimal128ColumnPrinter::Decimal128ColumnPrinter(std::string& buffer
- ): ColumnPrinter(buffer),
+ Decimal128ColumnPrinter::Decimal128ColumnPrinter(std::string& _buffer
+ ): ColumnPrinter(_buffer),
data(nullptr),
scale(0) {
// PASS
@@ -419,8 +419,8 @@ namespace orc {
}
}
- StringColumnPrinter::StringColumnPrinter(std::string& buffer
- ): ColumnPrinter(buffer),
+ StringColumnPrinter::StringColumnPrinter(std::string& _buffer
+ ): ColumnPrinter(_buffer),
start(nullptr),
length(nullptr) {
// PASS
@@ -470,9 +470,9 @@ namespace orc {
}
}
- ListColumnPrinter::ListColumnPrinter(std::string& buffer,
+ ListColumnPrinter::ListColumnPrinter(std::string& _buffer,
const Type& type
- ): ColumnPrinter(buffer),
+ ): ColumnPrinter(_buffer),
offsets(nullptr) {
elementPrinter = createColumnPrinter(buffer, type.getSubtype(0));
}
@@ -499,9 +499,9 @@ namespace orc {
}
}
- MapColumnPrinter::MapColumnPrinter(std::string& buffer,
+ MapColumnPrinter::MapColumnPrinter(std::string& _buffer,
const Type& type
- ): ColumnPrinter(buffer),
+ ): ColumnPrinter(_buffer),
offsets(nullptr) {
keyPrinter = createColumnPrinter(buffer, type.getSubtype(0));
elementPrinter = createColumnPrinter(buffer, type.getSubtype(1));
@@ -534,9 +534,9 @@ namespace orc {
}
}
- UnionColumnPrinter::UnionColumnPrinter(std::string& buffer,
+ UnionColumnPrinter::UnionColumnPrinter(std::string& _buffer,
const Type& type
- ): ColumnPrinter(buffer),
+ ): ColumnPrinter(_buffer),
tags(nullptr),
offsets(nullptr) {
for(unsigned int i=0; i < type.getSubtypeCount(); ++i) {
@@ -577,9 +577,9 @@ namespace orc {
}
}
- StructColumnPrinter::StructColumnPrinter(std::string& buffer,
+ StructColumnPrinter::StructColumnPrinter(std::string& _buffer,
const Type& type
- ): ColumnPrinter(buffer) {
+ ): ColumnPrinter(_buffer) {
for(unsigned int i=0; i < type.getSubtypeCount(); ++i) {
fieldNames.push_back(type.getFieldName(i));
fieldPrinter.push_back(createColumnPrinter(buffer,
@@ -621,8 +621,8 @@ namespace orc {
}
}
- DateColumnPrinter::DateColumnPrinter(std::string& buffer
- ): ColumnPrinter(buffer),
+ DateColumnPrinter::DateColumnPrinter(std::string& _buffer
+ ): ColumnPrinter(_buffer),
data(nullptr) {
// PASS
}
@@ -647,8 +647,8 @@ namespace orc {
data = dynamic_cast<const LongVectorBatch&>(batch).data.data();
}
- BooleanColumnPrinter::BooleanColumnPrinter(std::string& buffer
- ): ColumnPrinter(buffer),
+ BooleanColumnPrinter::BooleanColumnPrinter(std::string& _buffer
+ ): ColumnPrinter(_buffer),
data(nullptr) {
// PASS
}
@@ -666,8 +666,8 @@ namespace orc {
data = dynamic_cast<const LongVectorBatch&>(batch).data.data();
}
- BinaryColumnPrinter::BinaryColumnPrinter(std::string& buffer
- ): ColumnPrinter(buffer),
+ BinaryColumnPrinter::BinaryColumnPrinter(std::string& _buffer
+ ): ColumnPrinter(_buffer),
start(nullptr),
length(nullptr) {
// PASS
@@ -697,8 +697,8 @@ namespace orc {
length = dynamic_cast<const StringVectorBatch&>(batch).length.data();
}
- TimestampColumnPrinter::TimestampColumnPrinter(std::string& buffer
- ): ColumnPrinter(buffer),
+ TimestampColumnPrinter::TimestampColumnPrinter(std::string& _buffer
+ ): ColumnPrinter(_buffer),
seconds(nullptr),
nanoseconds(nullptr) {
// PASS
diff --git a/c++/src/Vector.cc b/c++/src/Vector.cc
index 9e7e2d3..8c8e773 100644
--- a/c++/src/Vector.cc
+++ b/c++/src/Vector.cc
@@ -56,9 +56,9 @@ namespace orc {
return false;
}
- LongVectorBatch::LongVectorBatch(uint64_t capacity, MemoryPool& pool
- ): ColumnVectorBatch(capacity, pool),
- data(pool, capacity) {
+ LongVectorBatch::LongVectorBatch(uint64_t _capacity, MemoryPool& pool
+ ): ColumnVectorBatch(_capacity, pool),
+ data(pool, _capacity) {
// PASS
}
@@ -84,9 +84,9 @@ namespace orc {
static_cast<uint64_t>(data.capacity() * sizeof(int64_t));
}
- DoubleVectorBatch::DoubleVectorBatch(uint64_t capacity, MemoryPool& pool
- ): ColumnVectorBatch(capacity, pool),
- data(pool, capacity) {
+ DoubleVectorBatch::DoubleVectorBatch(uint64_t _capacity, MemoryPool& pool
+ ): ColumnVectorBatch(_capacity, pool),
+ data(pool, _capacity) {
// PASS
}
@@ -112,10 +112,10 @@ namespace orc {
+ static_cast<uint64_t>(data.capacity() * sizeof(double));
}
- StringVectorBatch::StringVectorBatch(uint64_t capacity, MemoryPool& pool
- ): ColumnVectorBatch(capacity, pool),
- data(pool, capacity),
- length(pool, capacity) {
+ StringVectorBatch::StringVectorBatch(uint64_t _capacity, MemoryPool& pool
+ ): ColumnVectorBatch(_capacity, pool),
+ data(pool, _capacity),
+ length(pool, _capacity) {
// PASS
}
@@ -404,12 +404,12 @@ namespace orc {
return value.toDecimalString(scale);
}
- TimestampVectorBatch::TimestampVectorBatch(uint64_t capacity,
+ TimestampVectorBatch::TimestampVectorBatch(uint64_t _capacity,
MemoryPool& pool
- ): ColumnVectorBatch(capacity,
+ ): ColumnVectorBatch(_capacity,
pool),
- data(pool, capacity),
- nanoseconds(pool, capacity) {
+ data(pool, _capacity),
+ nanoseconds(pool, _capacity) {
// PASS
}
diff --git a/c++/test/TestColumnReader.cc b/c++/test/TestColumnReader.cc
index c67229a..9e1f356 100644
--- a/c++/test/TestColumnReader.cc
+++ b/c++/test/TestColumnReader.cc
@@ -32,6 +32,9 @@
#ifdef __clang__
DIAGNOSTIC_IGNORE("-Winconsistent-missing-override")
#endif
+#ifdef __GNUC__
+ DIAGNOSTIC_IGNORE("-Wparentheses")
+#endif
namespace orc {
diff --git a/cmake_modules/FindSnappy.cmake b/cmake_modules/FindSnappy.cmake
index 6cf64be..5aeb88e 100644
--- a/cmake_modules/FindSnappy.cmake
+++ b/cmake_modules/FindSnappy.cmake
@@ -30,7 +30,7 @@ find_path (SNAPPY_INCLUDE_DIR snappy.h HINTS
find_library (SNAPPY_LIBRARIES NAMES snappy HINTS
${_snappy_path}
- PATH_SUFFIXES "lib")
+ PATH_SUFFIXES "lib" "lib64")
if (SNAPPY_INCLUDE_DIR AND SNAPPY_LIBRARIES)
set (SNAPPY_FOUND TRUE)
diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake
index f8584c0..561fbc9 100644
--- a/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cmake_modules/ThirdpartyToolchain.cmake
@@ -59,7 +59,7 @@ else ()
set(SNAPPY_INCLUDE_DIR "${SNAPPY_HOME}/include")
set(SNAPPY_STATIC_LIB "${SNAPPY_HOME}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}snappy${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(SNAPPY_CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${SNAPPY_HOME}
- -DBUILD_SHARED_LIBS=OFF)
+ -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_LIBDIR=lib)
ExternalProject_Add (snappy_ep
URL "https://github.com/google/snappy/archive/${SNAPPY_VERSION}.tar.gz"