You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@parquet.apache.org by we...@apache.org on 2018/08/17 14:10:16 UTC
[parquet-cpp] branch master updated: PARQUET-1382: [C++] Prepare
for arrow::test namespace removal
This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/parquet-cpp.git
The following commit(s) were added to refs/heads/master by this push:
new ebb4b4b PARQUET-1382: [C++] Prepare for arrow::test namespace removal
ebb4b4b is described below
commit ebb4b4b219916f57ac6a817734bbce13dae12d6c
Author: Antoine Pitrou <an...@python.org>
AuthorDate: Fri Aug 17 10:10:09 2018 -0400
PARQUET-1382: [C++] Prepare for arrow::test namespace removal
Also tried to make Travis-CI builds faster.
Author: Antoine Pitrou <an...@python.org>
Closes #487 from pitrou/PARQUET-1382-arrow-test-namespace-shim and squashes the following commits:
2c169ec [Antoine Pitrou] Fix conda-forge brotli version, also exit early if conda fails
d47cde0 [Antoine Pitrou] Try to fix AppVeyor toolchain build
24a012a [Antoine Pitrou] Make tests a bit faster with Valgrind
180d32d [Antoine Pitrou] Only run with Valgrind in a single configuration
f87efb2 [Antoine Pitrou] Re-enable test verbosity, because of Valgrind slowness
625e62d [Antoine Pitrou] Try to silence deprecation warnings
8aff555 [Antoine Pitrou] Try to silence deprecation errors on gcc / clang
ec0297c [Antoine Pitrou] Bump Arrow changeset
5022828 [Antoine Pitrou] PARQUET-1382: [C++] Prepare for arrow::test namespace removal
---
.travis.yml | 15 ++++---
ci/before_script_travis.sh | 15 ++++---
ci/msvc-build.bat | 4 +-
ci/travis_script_cpp.sh | 8 ++--
ci/travis_script_static.sh | 14 +++++--
ci/travis_script_toolchain.sh | 16 +++++---
cmake_modules/ArrowExternalProject.cmake | 3 +-
cmake_modules/SetupCxxFlags.cmake | 7 ++--
src/parquet/arrow/arrow-reader-writer-test.cc | 56 ++++++++++-----------------
src/parquet/arrow/test-util.h | 28 +++++++++-----
src/parquet/bloom_filter-test.cc | 9 +++--
src/parquet/column_writer-test.cc | 8 ++++
12 files changed, 105 insertions(+), 78 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index 7918b89..e1faf68 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -14,8 +14,15 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+
sudo: required
dist: trusty
+
+language: cpp
+
+cache:
+ ccache: true
+
addons:
apt:
sources:
@@ -35,6 +42,7 @@ addons:
- bison
- flex
- pkg-config
+
matrix:
fast_finish: true
include:
@@ -42,10 +50,7 @@ matrix:
os: linux
before_script:
- export PARQUET_CXXFLAGS="-DARROW_NO_DEPRECATED_API"
- - source $TRAVIS_BUILD_DIR/ci/before_script_travis.sh
- - compiler: gcc
- os: linux
- before_script:
+ - export PARQUET_TRAVIS_VALGRIND=1
- source $TRAVIS_BUILD_DIR/ci/before_script_travis.sh
- compiler: clang
os: linux
@@ -76,8 +81,6 @@ matrix:
script:
- $TRAVIS_BUILD_DIR/ci/travis_script_toolchain.sh
-language: cpp
-
# PARQUET-626: revisit llvm toolchain when/if llvm.org apt repo resurfaces
# before_install:
diff --git a/ci/before_script_travis.sh b/ci/before_script_travis.sh
index 95a2fd8..ce0234c 100755
--- a/ci/before_script_travis.sh
+++ b/ci/before_script_travis.sh
@@ -28,15 +28,20 @@ fi
export PARQUET_TEST_DATA=$TRAVIS_BUILD_DIR/data
+CMAKE_COMMON_FLAGS="-DPARQUET_BUILD_WARNING_LEVEL=CHECKIN"
+
+if [ $PARQUET_TRAVIS_VALGRIND == "1" ]; then
+ CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DPARQUET_TEST_MEMCHECK=ON"
+fi
+
if [ $TRAVIS_OS_NAME == "linux" ]; then
- cmake -DPARQUET_CXXFLAGS="$PARQUET_CXXFLAGS" \
- -DPARQUET_TEST_MEMCHECK=ON \
+ cmake $CMAKE_COMMON_FLAGS \
+ -DPARQUET_CXXFLAGS="$PARQUET_CXXFLAGS" \
-DPARQUET_BUILD_BENCHMARKS=ON \
- -DPARQUET_BUILD_WARNING_LEVEL=CHECKIN \
-DPARQUET_GENERATE_COVERAGE=1 \
$TRAVIS_BUILD_DIR
else
- cmake -DPARQUET_CXXFLAGS="$PARQUET_CXXFLAGS" \
- -DPARQUET_BUILD_WARNING_LEVEL=CHECKIN \
+ cmake $CMAKE_COMMON_FLAGS \
+ -DPARQUET_CXXFLAGS="$PARQUET_CXXFLAGS" \
$TRAVIS_BUILD_DIR
fi
diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat
index 0136819..7a50c85 100644
--- a/ci/msvc-build.bat
+++ b/ci/msvc-build.bat
@@ -45,8 +45,8 @@ if defined need_vcvarsall (
if "%CONFIGURATION%" == "Toolchain" (
conda install -y boost-cpp=1.63 thrift-cpp=0.11.0 ^
- brotli=0.6.0 zlib=1.2.11 snappy=1.1.6 lz4-c=1.7.5 zstd=1.2.0 ^
- -c conda-forge
+ brotli=1.0.2 zlib=1.2.11 snappy=1.1.7 lz4-c=1.8.0 zstd=1.3.3 ^
+ -c conda-forge || exit /B
set ARROW_BUILD_TOOLCHAIN=%MINICONDA%/Library
set PARQUET_BUILD_TOOLCHAIN=%MINICONDA%/Library
diff --git a/ci/travis_script_cpp.sh b/ci/travis_script_cpp.sh
index d3cef66..3031363 100755
--- a/ci/travis_script_cpp.sh
+++ b/ci/travis_script_cpp.sh
@@ -33,18 +33,18 @@ make lint
# fi
if [ $TRAVIS_OS_NAME == "linux" ]; then
- make -j4 || exit 1
- ctest -VV -L unittest || { cat $TRAVIS_BUILD_DIR/parquet-build/Testing/Temporary/LastTest.log; exit 1; }
+ make -j4
+ ctest -j2 -VV -L unittest
# Current cpp-coveralls version 0.4 throws an error (PARQUET-1075) on Travis CI. Pin to last working version
sudo pip install cpp_coveralls==0.3.12
export PARQUET_ROOT=$TRAVIS_BUILD_DIR
$TRAVIS_BUILD_DIR/ci/upload_coverage.sh
else
- make -j4 || exit 1
+ make -j4
BUILD_TYPE=debug
EXECUTABLE_DIR=$CPP_BUILD_DIR/$BUILD_TYPE
export LD_LIBRARY_PATH=$EXECUTABLE_DIR:$LD_LIBRARY_PATH
- ctest -VV -L unittest || { cat $TRAVIS_BUILD_DIR/parquet-build/Testing/Temporary/LastTest.log; exit 1; }
+ ctest -j2 -VV -L unittest
fi
popd
diff --git a/ci/travis_script_static.sh b/ci/travis_script_static.sh
index b76ced8..8af574e 100755
--- a/ci/travis_script_static.sh
+++ b/ci/travis_script_static.sh
@@ -65,8 +65,14 @@ export ZLIB_STATIC_LIB=$ARROW_EP/zlib_ep/src/zlib_ep-install/lib/libz.a
export LZ4_STATIC_LIB=$ARROW_EP/lz4_ep-prefix/src/lz4_ep/lib/liblz4.a
export ZSTD_STATIC_LIB=$ARROW_EP/zstd_ep-prefix/src/zstd_ep/lib/libzstd.a
-cmake -DPARQUET_CXXFLAGS="$PARQUET_CXXFLAGS" \
- -DPARQUET_TEST_MEMCHECK=ON \
+CMAKE_COMMON_FLAGS="-DPARQUET_BUILD_WARNING_LEVEL=CHECKIN"
+
+if [ $PARQUET_TRAVIS_VALGRIND == "1" ]; then
+ CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DPARQUET_TEST_MEMCHECK=ON"
+fi
+
+cmake $CMAKE_COMMON_FLAGS \
+ -DPARQUET_CXXFLAGS="$PARQUET_CXXFLAGS" \
-DPARQUET_ARROW_LINKAGE="static" \
-DPARQUET_BUILD_SHARED=OFF \
-DPARQUET_BOOST_USE_SHARED=OFF \
@@ -78,7 +84,7 @@ cmake -DPARQUET_CXXFLAGS="$PARQUET_CXXFLAGS" \
pushd $CPP_BUILD_DIR
-make -j4 VERBOSE=1 || exit 1
-ctest -VV -L unittest || { cat $TRAVIS_BUILD_DIR/parquet-build/Testing/Temporary/LastTest.log; exit 1; }
+make -j4 VERBOSE=1
+ctest -j2 -VV -L unittest
popd
diff --git a/ci/travis_script_toolchain.sh b/ci/travis_script_toolchain.sh
index 036ab3c..e94552c 100755
--- a/ci/travis_script_toolchain.sh
+++ b/ci/travis_script_toolchain.sh
@@ -51,17 +51,23 @@ export PARQUET_BUILD_TOOLCHAIN=$CPP_TOOLCHAIN
export LD_LIBRARY_PATH=$CPP_TOOLCHAIN/lib:$LD_LIBRARY_PATH
export BOOST_ROOT=$CPP_TOOLCHAIN
-cmake -DPARQUET_CXXFLAGS=-Werror \
- -DPARQUET_TEST_MEMCHECK=ON \
+CMAKE_COMMON_FLAGS="-DPARQUET_BUILD_WARNING_LEVEL=CHECKIN"
+
+if [ $PARQUET_TRAVIS_VALGRIND == "1" ]; then
+ CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DPARQUET_TEST_MEMCHECK=ON"
+fi
+
+cmake $CMAKE_COMMON_FLAGS \
+ -DPARQUET_CXXFLAGS=-Werror \
-DPARQUET_GENERATE_COVERAGE=1 \
-DCMAKE_INSTALL_PREFIX=$CPP_TOOLCHAIN \
$TRAVIS_BUILD_DIR
pushd $CPP_BUILD_DIR
-make -j4 || exit 1
-make install || exit 1
-ctest -VV -L unittest || { cat $TRAVIS_BUILD_DIR/parquet-build/Testing/Temporary/LastTest.log; exit 1; }
+make -j4
+make install
+ctest -j2 -VV -L unittest
popd
diff --git a/cmake_modules/ArrowExternalProject.cmake b/cmake_modules/ArrowExternalProject.cmake
index 3d1a276..51eeca5 100644
--- a/cmake_modules/ArrowExternalProject.cmake
+++ b/cmake_modules/ArrowExternalProject.cmake
@@ -46,7 +46,8 @@ if (MSVC AND PARQUET_USE_STATIC_CRT)
endif()
if ("$ENV{PARQUET_ARROW_VERSION}" STREQUAL "")
- set(ARROW_VERSION "3edfd7caf2746eeba37d5ac7bfd3665cc159e7ad")
+ # This can be a tag or changeset
+ set(ARROW_VERSION "170dc75468efbad2286c630b9103d1aacdb6bada")
else()
set(ARROW_VERSION "$ENV{PARQUET_ARROW_VERSION}")
endif()
diff --git a/cmake_modules/SetupCxxFlags.cmake b/cmake_modules/SetupCxxFlags.cmake
index 5ca3f4e..0029501 100644
--- a/cmake_modules/SetupCxxFlags.cmake
+++ b/cmake_modules/SetupCxxFlags.cmake
@@ -79,8 +79,8 @@ if ("${UPPERCASE_BUILD_WARNING_LEVEL}" STREQUAL "CHECKIN")
set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /WX")
elseif ("${COMPILER_FAMILY}" STREQUAL "clang")
set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Weverything -Wno-c++98-compat \
--Wno-c++98-compat-pedantic -Wno-deprecated -Wno-weak-vtables -Wno-padded \
--Wno-comma -Wno-unused-parameter -Wno-undef \
+-Wno-c++98-compat-pedantic -Wno-deprecated -Wno-deprecated-declarations \
+-Wno-weak-vtables -Wno-padded -Wno-comma -Wno-unused-parameter -Wno-undef \
-Wno-shadow -Wno-switch-enum -Wno-exit-time-destructors \
-Wno-global-constructors -Wno-weak-template-vtables -Wno-undefined-reinterpret-cast \
-Wno-implicit-fallthrough -Wno-unreachable-code-return \
@@ -118,7 +118,8 @@ if ("${UPPERCASE_BUILD_WARNING_LEVEL}" STREQUAL "CHECKIN")
# Treat all compiler warnings as errors
set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-unknown-warning-option -Werror")
elseif ("${COMPILER_FAMILY}" STREQUAL "gcc")
- set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wall -Wconversion -Wno-sign-conversion")
+ set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wall \
+-Wno-deprecated-declarations -Wconversion -Wno-sign-conversion")
# Treat all compiler warnings as errors
set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Werror")
else()
diff --git a/src/parquet/arrow/arrow-reader-writer-test.cc b/src/parquet/arrow/arrow-reader-writer-test.cc
index be3e611..ad78c22 100644
--- a/src/parquet/arrow/arrow-reader-writer-test.cc
+++ b/src/parquet/arrow/arrow-reader-writer-test.cc
@@ -64,8 +64,8 @@ using arrow::compute::DictionaryEncode;
using arrow::compute::FunctionContext;
using arrow::io::BufferReader;
-using arrow::test::randint;
-using arrow::test::random_is_valid;
+using arrow::randint;
+using arrow::random_is_valid;
using ArrowId = ::arrow::Type;
using ParquetType = parquet::Type;
@@ -80,7 +80,11 @@ namespace parquet {
namespace arrow {
static constexpr int SMALL_SIZE = 100;
+#ifdef PARQUET_VALGRIND
+static constexpr int LARGE_SIZE = 1000;
+#else
static constexpr int LARGE_SIZE = 10000;
+#endif
static constexpr uint32_t kDefaultSeed = 0;
@@ -375,29 +379,6 @@ void PrintColumn(const Column& col, std::stringstream* ss) {
}
}
-void AssertTablesEqual(const Table& expected, const Table& actual,
- bool same_chunk_layout = true) {
- ASSERT_EQ(expected.num_columns(), actual.num_columns());
-
- if (same_chunk_layout) {
- for (int i = 0; i < actual.num_columns(); ++i) {
- AssertChunkedEqual(*expected.column(i)->data(), *actual.column(i)->data());
- }
- } else {
- std::stringstream ss;
- if (!actual.Equals(expected)) {
- for (int i = 0; i < expected.num_columns(); ++i) {
- ss << "Actual column " << i << std::endl;
- PrintColumn(*actual.column(i), &ss);
-
- ss << "Expected column " << i << std::endl;
- PrintColumn(*expected.column(i), &ss);
- }
- FAIL() << ss.str();
- }
- }
-}
-
void DoSimpleRoundtrip(const std::shared_ptr<Table>& table, bool use_threads,
int64_t row_group_size, const std::vector<int>& column_subset,
std::shared_ptr<Table>* out,
@@ -428,7 +409,7 @@ void CheckSimpleRoundtrip(const std::shared_ptr<Table>& table, int64_t row_group
std::shared_ptr<Table> result;
DoSimpleRoundtrip(table, false /* use_threads */, row_group_size, {}, &result,
arrow_properties);
- ASSERT_NO_FATAL_FAILURE(AssertTablesEqual(*table, *result, false));
+ ASSERT_NO_FATAL_FAILURE(::arrow::AssertTablesEqual(*table, *result, false));
}
static std::shared_ptr<GroupNode> MakeSimpleSchema(const ::DataType& type,
@@ -1273,7 +1254,7 @@ TEST(TestArrowReadWrite, DateTimeTypes) {
table, false /* use_threads */, table->num_rows(), {}, &result,
ArrowWriterProperties::Builder().enable_deprecated_int96_timestamps()->build()));
- ASSERT_NO_FATAL_FAILURE(AssertTablesEqual(*table, *result));
+ ASSERT_NO_FATAL_FAILURE(::arrow::AssertTablesEqual(*table, *result));
// Cast nanaoseconds to microseconds and use INT64 physical type
ASSERT_NO_FATAL_FAILURE(
@@ -1281,7 +1262,7 @@ TEST(TestArrowReadWrite, DateTimeTypes) {
std::shared_ptr<Table> expected;
MakeDateTimeTypesTable(&table, true);
- ASSERT_NO_FATAL_FAILURE(AssertTablesEqual(*table, *result));
+ ASSERT_NO_FATAL_FAILURE(::arrow::AssertTablesEqual(*table, *result));
}
TEST(TestArrowReadWrite, CoerceTimestamps) {
@@ -1343,13 +1324,13 @@ TEST(TestArrowReadWrite, CoerceTimestamps) {
input, false /* use_threads */, input->num_rows(), {}, &milli_result,
ArrowWriterProperties::Builder().coerce_timestamps(TimeUnit::MILLI)->build()));
- ASSERT_NO_FATAL_FAILURE(AssertTablesEqual(*ex_milli_result, *milli_result));
+ ASSERT_NO_FATAL_FAILURE(::arrow::AssertTablesEqual(*ex_milli_result, *milli_result));
std::shared_ptr<Table> micro_result;
ASSERT_NO_FATAL_FAILURE(DoSimpleRoundtrip(
input, false /* use_threads */, input->num_rows(), {}, µ_result,
ArrowWriterProperties::Builder().coerce_timestamps(TimeUnit::MICRO)->build()));
- ASSERT_NO_FATAL_FAILURE(AssertTablesEqual(*ex_micro_result, *micro_result));
+ ASSERT_NO_FATAL_FAILURE(::arrow::AssertTablesEqual(*ex_micro_result, *micro_result));
}
TEST(TestArrowReadWrite, CoerceTimestampsLosePrecision) {
@@ -1476,7 +1457,7 @@ TEST(TestArrowReadWrite, ConvertedDateTimeTypes) {
ASSERT_NO_FATAL_FAILURE(
DoSimpleRoundtrip(table, false /* use_threads */, table->num_rows(), {}, &result));
- ASSERT_NO_FATAL_FAILURE(AssertTablesEqual(*ex_table, *result));
+ ASSERT_NO_FATAL_FAILURE(::arrow::AssertTablesEqual(*ex_table, *result));
}
// Regression for ARROW-2802
@@ -1612,7 +1593,7 @@ TEST(TestArrowReadWrite, MultithreadedRead) {
ASSERT_NO_FATAL_FAILURE(
DoSimpleRoundtrip(table, use_threads, table->num_rows(), {}, &result));
- ASSERT_NO_FATAL_FAILURE(AssertTablesEqual(*table, *result));
+ ASSERT_NO_FATAL_FAILURE(::arrow::AssertTablesEqual(*table, *result));
}
TEST(TestArrowReadWrite, ReadSingleRowGroup) {
@@ -1723,7 +1704,7 @@ TEST(TestArrowReadWrite, ReadColumnSubset) {
auto ex_schema = ::arrow::schema(ex_fields);
auto expected = Table::Make(ex_schema, ex_columns);
- ASSERT_NO_FATAL_FAILURE(AssertTablesEqual(*expected, *result));
+ ASSERT_NO_FATAL_FAILURE(::arrow::AssertTablesEqual(*expected, *result));
}
TEST(TestArrowReadWrite, ListLargeRecords) {
@@ -1751,7 +1732,7 @@ TEST(TestArrowReadWrite, ListLargeRecords) {
// Read everything
std::shared_ptr<Table> result;
ASSERT_OK_NO_THROW(reader->ReadTable(&result));
- ASSERT_NO_FATAL_FAILURE(AssertTablesEqual(*table, *result));
+ ASSERT_NO_FATAL_FAILURE(::arrow::AssertTablesEqual(*table, *result));
// Read chunked
ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
@@ -1927,7 +1908,7 @@ TEST(TestArrowReadWrite, DictionaryColumnChunkedWrite) {
auto expected_table = Table::Make(schema, columns);
- AssertTablesEqual(*expected_table, *result, false);
+ ::arrow::AssertTablesEqual(*expected_table, *result, false);
}
TEST(TestArrowWrite, CheckChunkSize) {
@@ -2276,8 +2257,13 @@ TEST_F(TestNestedSchemaRead, StructAndListTogetherUnsupported) {
}
TEST_P(TestNestedSchemaRead, DeepNestedSchemaRead) {
+#ifdef PARQUET_VALGRIND
+ const int num_trees = 3;
+ const int depth = 3;
+#else
const int num_trees = 10;
const int depth = 5;
+#endif
const int num_children = 3;
int num_rows = SMALL_SIZE * (depth + 2);
ASSERT_NO_FATAL_FAILURE(CreateMultiLevelNestedParquet(num_trees, depth, num_children,
diff --git a/src/parquet/arrow/test-util.h b/src/parquet/arrow/test-util.h
index f0f9139..19837db 100644
--- a/src/parquet/arrow/test-util.h
+++ b/src/parquet/arrow/test-util.h
@@ -26,6 +26,14 @@
#include "parquet/arrow/record_reader.h"
+namespace arrow {
+// PARQUET-1382: backwards-compatible shim for arrow::test namespace
+namespace test {}
+
+using namespace ::arrow::test; // NOLINT
+
+} // namespace arrow
+
namespace parquet {
using internal::RecordReader;
@@ -71,7 +79,7 @@ typename std::enable_if<is_arrow_float<ArrowType>::value, Status>::type NonNullA
size_t size, std::shared_ptr<Array>* out) {
using c_type = typename ArrowType::c_type;
std::vector<c_type> values;
- ::arrow::test::random_real(size, 0, static_cast<c_type>(0), static_cast<c_type>(1),
+ ::arrow::random_real(size, 0, static_cast<c_type>(0), static_cast<c_type>(1),
&values);
::arrow::NumericBuilder<ArrowType> builder;
RETURN_NOT_OK(builder.AppendValues(values.data(), values.size()));
@@ -83,7 +91,7 @@ typename std::enable_if<
is_arrow_int<ArrowType>::value && !is_arrow_date<ArrowType>::value, Status>::type
NonNullArray(size_t size, std::shared_ptr<Array>* out) {
std::vector<typename ArrowType::c_type> values;
- ::arrow::test::randint(size, 0, 64, &values);
+ ::arrow::randint(size, 0, 64, &values);
// Passing data type so this will work with TimestampType too
::arrow::NumericBuilder<ArrowType> builder(std::make_shared<ArrowType>(),
@@ -96,7 +104,7 @@ template <class ArrowType>
typename std::enable_if<is_arrow_date<ArrowType>::value, Status>::type NonNullArray(
size_t size, std::shared_ptr<Array>* out) {
std::vector<typename ArrowType::c_type> values;
- ::arrow::test::randint(size, 0, 64, &values);
+ ::arrow::randint(size, 0, 64, &values);
for (size_t i = 0; i < size; i++) {
values[i] *= 86400000;
}
@@ -180,7 +188,7 @@ template <class ArrowType>
typename std::enable_if<is_arrow_bool<ArrowType>::value, Status>::type NonNullArray(
size_t size, std::shared_ptr<Array>* out) {
std::vector<uint8_t> values;
- ::arrow::test::randint(size, 0, 1, &values);
+ ::arrow::randint(size, 0, 1, &values);
::arrow::BooleanBuilder builder;
RETURN_NOT_OK(builder.AppendValues(values.data(), values.size()));
return builder.Finish(out);
@@ -192,7 +200,7 @@ typename std::enable_if<is_arrow_float<ArrowType>::value, Status>::type Nullable
size_t size, size_t num_nulls, uint32_t seed, std::shared_ptr<Array>* out) {
using c_type = typename ArrowType::c_type;
std::vector<c_type> values;
- ::arrow::test::random_real(size, seed, static_cast<c_type>(-1e10),
+ ::arrow::random_real(size, seed, static_cast<c_type>(-1e10),
static_cast<c_type>(1e10), &values);
std::vector<uint8_t> valid_bytes(size, 1);
@@ -214,7 +222,7 @@ NullableArray(size_t size, size_t num_nulls, uint32_t seed, std::shared_ptr<Arra
// Seed is random in Arrow right now
(void)seed;
- ::arrow::test::randint(size, 0, 64, &values);
+ ::arrow::randint(size, 0, 64, &values);
std::vector<uint8_t> valid_bytes(size, 1);
for (size_t i = 0; i < num_nulls; i++) {
@@ -235,7 +243,7 @@ typename std::enable_if<is_arrow_date<ArrowType>::value, Status>::type NullableA
// Seed is random in Arrow right now
(void)seed;
- ::arrow::test::randint(size, 0, 64, &values);
+ ::arrow::randint(size, 0, 64, &values);
for (size_t i = 0; i < size; i++) {
values[i] *= 86400000;
}
@@ -273,7 +281,7 @@ NullableArray(size_t size, size_t num_nulls, uint32_t seed,
if (!valid_bytes[i]) {
RETURN_NOT_OK(builder.AppendNull());
} else {
- ::arrow::test::random_bytes(kBufferSize, seed + static_cast<uint32_t>(i), buffer);
+ ::arrow::random_bytes(kBufferSize, seed + static_cast<uint32_t>(i), buffer);
RETURN_NOT_OK(builder.Append(buffer, kBufferSize));
}
}
@@ -302,7 +310,7 @@ NullableArray(size_t size, size_t num_nulls, uint32_t seed,
if (!valid_bytes[i]) {
RETURN_NOT_OK(builder.AppendNull());
} else {
- ::arrow::test::random_bytes(kBufferSize, seed + static_cast<uint32_t>(i), buffer);
+ ::arrow::random_bytes(kBufferSize, seed + static_cast<uint32_t>(i), buffer);
RETURN_NOT_OK(builder.Append(buffer));
}
}
@@ -346,7 +354,7 @@ typename std::enable_if<is_arrow_bool<ArrowType>::value, Status>::type NullableA
// Seed is random in Arrow right now
(void)seed;
- ::arrow::test::randint(size, 0, 1, &values);
+ ::arrow::randint(size, 0, 1, &values);
std::vector<uint8_t> valid_bytes(size, 1);
for (size_t i = 0; i < num_nulls; i++) {
diff --git a/src/parquet/bloom_filter-test.cc b/src/parquet/bloom_filter-test.cc
index dbef8c8..69583af 100644
--- a/src/parquet/bloom_filter-test.cc
+++ b/src/parquet/bloom_filter-test.cc
@@ -72,7 +72,7 @@ TEST(BasicTest, TestBloomFilter) {
// Deserialize Bloom filter from memory
InMemoryInputStream source(sink.GetBuffer());
- BlockSplitBloomFilter de_bloom = std::move(BlockSplitBloomFilter::Deserialize(&source));
+ BlockSplitBloomFilter de_bloom = BlockSplitBloomFilter::Deserialize(&source);
for (int i = 0; i < 10; i++) {
EXPECT_TRUE(de_bloom.FindHash(de_bloom.Hash(i)));
@@ -104,7 +104,11 @@ TEST(FPPTest, TestBloomFilter) {
int exist = 0;
// Total count of elements that will be used
+#ifdef PARQUET_VALGRIND
+ const int total_count = 5000;
+#else
const int total_count = 100000;
+#endif
// Bloom filter fpp parameter
const double fpp = 0.01;
@@ -164,8 +168,7 @@ TEST(CompatibilityTest, TestBloomFilter) {
handle->Read(size, &buffer);
InMemoryInputStream source(buffer);
- BlockSplitBloomFilter bloom_filter1 =
- std::move(BlockSplitBloomFilter::Deserialize(&source));
+ BlockSplitBloomFilter bloom_filter1 = BlockSplitBloomFilter::Deserialize(&source);
for (int i = 0; i < 4; i++) {
const ByteArray tmp(static_cast<uint32_t>(test_string[i].length()),
diff --git a/src/parquet/column_writer-test.cc b/src/parquet/column_writer-test.cc
index 6c0794a..dd89281 100644
--- a/src/parquet/column_writer-test.cc
+++ b/src/parquet/column_writer-test.cc
@@ -35,10 +35,18 @@ namespace test {
// The default size used in most tests.
const int SMALL_SIZE = 100;
+#ifdef PARQUET_VALGRIND
+// Larger size to test some corner cases, only used in some specific cases.
+const int LARGE_SIZE = 10000;
+// Very large size to test dictionary fallback.
+const int VERY_LARGE_SIZE = 40000;
+#else
// Larger size to test some corner cases, only used in some specific cases.
const int LARGE_SIZE = 100000;
// Very large size to test dictionary fallback.
const int VERY_LARGE_SIZE = 400000;
+#endif
+
template <typename TestType>
class TestPrimitiveWriter : public PrimitiveTypedTest<TestType> {