You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2019/07/08 18:55:15 UTC
[arrow] branch master updated: ARROW-5525: [C++] Add Continuous
Fuzzing Integration setup with Fuzzit
This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 10890c8 ARROW-5525: [C++] Add Continuous Fuzzing Integration setup with Fuzzit
10890c8 is described below
commit 10890c8f0881b2fb03da024375cd1145aad9d4f8
Author: Jeka Pats <ye...@gmail.com>
AuthorDate: Mon Jul 8 13:54:59 2019 -0500
ARROW-5525: [C++] Add Continuous Fuzzing Integration setup with Fuzzit
Adds fuzzit test to docker-compose and dev/tasks/tests.yml
Author: Jeka Pats <ye...@gmail.com>
Closes #4504 from yevgenypats/ARROW-5525 and squashes the following commits:
7f10bb744 <Jeka Pats> Migrate Fuzzit Continuous Fuzzing to new infra (dockerized)
6380a3ce4 <Jeka Pats> Continuous fuzzing integration with fuzzit
3079aeea0 <Jeka Pats> WIP: Continuous Fuzzing Integration with Fuzzit
---
README.md | 1 +
ci/docker_build_and_fuzzit.sh | 48 +++++++++++++++++++++++++++++++++++++++++++
ci/docker_build_cpp.sh | 2 ++
dev/fuzzit/Dockerfile | 28 +++++++++++++++++++++++++
dev/tasks/tests.yml | 11 ++++++++++
docker-compose.yml | 11 ++++++++++
6 files changed, 101 insertions(+)
diff --git a/README.md b/README.md
index 24157b3..ff8f52d 100644
--- a/README.md
+++ b/README.md
@@ -21,6 +21,7 @@
[](https://travis-ci.org/apache/arrow)
[](https://codecov.io/gh/apache/arrow?branch=master)
+[](https://app.fuzzit.dev/admin/yMxZh42xl9qy6bvg3EiJ/dashboard)
[](https://github.com/apache/arrow/blob/master/LICENSE.txt)
[](https://twitter.com/apachearrow)
diff --git a/ci/docker_build_and_fuzzit.sh b/ci/docker_build_and_fuzzit.sh
new file mode 100755
index 0000000..3ab3145
--- /dev/null
+++ b/ci/docker_build_and_fuzzit.sh
@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+export ARROW_FUZZING="ON"
+#export ARROW_DEPENDENCY_SOURCE="BUNDLED"
+export ARROW_USE_ASAN="ON"
+export CC="clang-7"
+export CXX="clang++-7"
+export ARROW_BUILD_TYPE="RelWithDebInfo"
+export ARROW_ORC="OFF"
+export ARROW_PARQUET="OFF"
+export ARROW_PLASMA="OFF"
+export ARROW_FLIGHT="OFF"
+export ARROW_BUILD_BENCHMARKS="OFF"
+export ARROW_WITH_BZ2="OFF"
+export ARROW_WITH_ZSTD="OFF"
+export ARROW_BUILD_UTILITIES="OFF"
+/arrow/ci/docker_build_cpp.sh || exit 1
+pushd /build/cpp
+
+mkdir ./relwithdebinfo/out
+cp ./relwithdebinfo/arrow-ipc-fuzzing-test ./relwithdebinfo/out/fuzzer
+ldd ./relwithdebinfo/arrow-ipc-fuzzing-test | grep "=> /" | awk '{print $3}' | xargs -I '{}' cp -v '{}' ./relwithdebinfo/out/.
+cd ./relwithdebinfo/out/
+tar -czvf fuzzer.tar.gz *
+cd ../../
+
+export TARGET_ID=u79f6bXYgNH4NkU99iWK
+export FUZZIT_API_KEY=${FUZZIT_API_KEY:-ac6089a1bc2313679f2d99bb80553162c380676bff3f094de826b16229e28184a8084b86f52c95112bde6b3dbb07b9b7}
+wget -O fuzzit https://bin.fuzzit.dev/fuzzit-1.1
+chmod a+x fuzzit
+./fuzzit auth $FUZZIT_API_KEY
+./fuzzit create job --type fuzzing --host bionic-llvm7 --revision $CIRCLE_SHA1 --branch $CIRCLE_BRANCH $TARGET_ID ./relwithdebinfo/out/fuzzer.tar.gz
diff --git a/ci/docker_build_cpp.sh b/ci/docker_build_cpp.sh
index 471a732..77268bc 100755
--- a/ci/docker_build_cpp.sh
+++ b/ci/docker_build_cpp.sh
@@ -56,6 +56,8 @@ cmake -GNinja \
-DCMAKE_CXX_FLAGS=$CXXFLAGS \
-DBOOST_SOURCE=${ARROW_BOOST_SOURCE:-AUTO} \
-Duriparser_SOURCE=AUTO \
+ -DARROW_FUZZING=${ARROW_FUZZING:-OFF} \
+ -DARROW_USE_ASAN=${ARROW_USE_ASAN:-OFF} \
${CMAKE_ARGS} \
${source_dir}
ninja
diff --git a/dev/fuzzit/Dockerfile b/dev/fuzzit/Dockerfile
new file mode 100644
index 0000000..602cb5f
--- /dev/null
+++ b/dev/fuzzit/Dockerfile
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM arrow:cpp
+
+RUN apt-get update && \
+ apt-get install -y -q \
+ clang-7 \
+ libclang-7-dev \
+ clang-format-7 \
+ clang-tidy-7 \
+ clang-tools-7
+
+CMD ["arrow/ci/docker_build_and_fuzzit.sh"]
diff --git a/dev/tasks/tests.yml b/dev/tasks/tests.yml
index 2c424fa..32e2ed5 100644
--- a/dev/tasks/tests.yml
+++ b/dev/tasks/tests.yml
@@ -25,6 +25,7 @@ groups:
- docker-cpp-alpine
- docker-cpp-cmake32
- docker-cpp-release
+ - docker-cpp-fuzzit
- docker-c_glib
- docker-go
- docker-python-2.7
@@ -132,6 +133,16 @@ tasks:
- docker-compose build cpp-cmake32
- docker-compose run cpp-cmake32
+ docker-cpp-fuzzit:
+ ci: circle
+ platform: linux
+ template: docker-tests/circle.linux.yml
+ params:
+ commands:
+ - docker-compose build cpp
+ - docker-compose build fuzzit
+ - docker-compose run fuzzit
+
docker-c_glib:
ci: circle
platform: linux
diff --git a/docker-compose.yml b/docker-compose.yml
index 768c4e6..0b977b1 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -398,6 +398,17 @@ services:
command: arrow/dev/lint/run_linters.sh
volumes: *ubuntu-volumes
+ fuzzit:
+ # Usage:
+ # docker-compose build cpp
+ # docker-compose build fuzzit
+ # docker-compose run fuzzit
+ image: arrow:fuzzit
+ build:
+ context: .
+ dockerfile: dev/fuzzit/Dockerfile
+ volumes: *ubuntu-volumes
+
iwyu:
# Usage:
# export PYTHON_VERSION=3.6