You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by np...@apache.org on 2020/06/24 15:55:52 UTC
[arrow] branch master updated: ARROW-6945: [Rust][Integration] Run
rust integration tests
This is an automated email from the ASF dual-hosted git repository.
npr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 8b571fd ARROW-6945: [Rust][Integration] Run rust integration tests
8b571fd is described below
commit 8b571fdf4881fbe95c29f93b703246c5f864f81d
Author: Neal Richardson <ne...@gmail.com>
AuthorDate: Wed Jun 24 08:55:25 2020 -0700
ARROW-6945: [Rust][Integration] Run rust integration tests
FYI @andygrove. I looked at the integration test output and saw that Rust tests weren't being run at all, so I'm trying to enable them.
Closes #7297 from nealrichardson/run-rust-integration
Lead-authored-by: Neal Richardson <ne...@gmail.com>
Co-authored-by: Wakahisa <ne...@gmail.com>
Co-authored-by: Neville Dipale <ne...@gmail.com>
Signed-off-by: Neal Richardson <ne...@gmail.com>
---
.github/workflows/integration.yml | 2 ++
ci/docker/conda-integration.dockerfile | 2 ++
ci/scripts/rust_build.sh | 5 -----
dev/archery/archery/cli.py | 2 +-
dev/archery/archery/integration/datagen.py | 20 ++++++++++++++------
docker-compose.yml | 3 ++-
.../src/bin/arrow-json-integration-test.rs | 22 ++++++++++++++++++++--
7 files changed, 41 insertions(+), 15 deletions(-)
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
index 905ff01..ead90ef 100644
--- a/.github/workflows/integration.yml
+++ b/.github/workflows/integration.yml
@@ -29,6 +29,7 @@ on:
- 'cpp/**'
- 'java/**'
- 'format/**'
+ - 'rust/**'
pull_request:
paths:
- '.github/workflows/integration.yml'
@@ -40,6 +41,7 @@ on:
- 'cpp/**'
- 'java/**'
- 'format/**'
+ - 'rust/**'
env:
DOCKER_BUILDKIT: 0
diff --git a/ci/docker/conda-integration.dockerfile b/ci/docker/conda-integration.dockerfile
index 5672cb8..088d93e 100644
--- a/ci/docker/conda-integration.dockerfile
+++ b/ci/docker/conda-integration.dockerfile
@@ -34,6 +34,8 @@ RUN conda install -q \
openjdk=${jdk} && \
conda clean --all
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+
ENV GOROOT=/opt/go \
GOBIN=/opt/go/bin \
GOPATH=/go \
diff --git a/ci/scripts/rust_build.sh b/ci/scripts/rust_build.sh
index 67305cd..7bbef28 100755
--- a/ci/scripts/rust_build.sh
+++ b/ci/scripts/rust_build.sh
@@ -20,19 +20,14 @@
set -ex
source_dir=${1}/rust
-build_dir=${2}/rust
export ARROW_TEST_DATA=${arrow_dir}/testing/data
export PARQUET_TEST_DATA=${arrow_dir}/cpp/submodules/parquet-testing/data
-export CARGO_TARGET_DIR=${build_dir}
export RUSTFLAGS="-D warnings"
# show activated toolchain
rustup show
-# ensure that the build directory exists
-mkdir -p ${build_dir}
-
pushd ${source_dir}
# build entire project
diff --git a/dev/archery/archery/cli.py b/dev/archery/archery/cli.py
index 2622fbf..7a947df 100644
--- a/dev/archery/archery/cli.py
+++ b/dev/archery/archery/cli.py
@@ -651,7 +651,7 @@ def integration(with_all=False, random_seed=12345, **args):
gen_path = args['write_generated_json']
- languages = ['cpp', 'java', 'js', 'go']
+ languages = ['cpp', 'java', 'js', 'go', 'rust']
enabled_languages = 0
for lang in languages:
diff --git a/dev/archery/archery/integration/datagen.py b/dev/archery/archery/integration/datagen.py
index 7c0b574..6a23d0b 100644
--- a/dev/archery/archery/integration/datagen.py
+++ b/dev/archery/archery/integration/datagen.py
@@ -1487,18 +1487,23 @@ def get_generated_json_files(tempdir=None, flight=False):
file_objs = [
generate_primitive_case([], name='primitive_no_batches'),
- generate_primitive_case([17, 20], name='primitive'),
- generate_primitive_case([0, 0, 0], name='primitive_zerolength'),
+ generate_primitive_case([17, 20], name='primitive')
+ .skip_category('Rust'),
+ generate_primitive_case([0, 0, 0], name='primitive_zerolength')
+ .skip_category('Rust'),
generate_primitive_large_offsets_case([17, 20])
.skip_category('Go')
- .skip_category('JS'),
+ .skip_category('JS')
+ .skip_category('Rust'),
generate_null_case([10, 0])
+ .skip_category('Rust')
.skip_category('JS') # TODO(ARROW-7900)
.skip_category('Go'), # TODO(ARROW-7901)
generate_null_trivial_case([0, 0])
+ .skip_category('Rust')
.skip_category('JS') # TODO(ARROW-7900)
.skip_category('Go'), # TODO(ARROW-7901)
@@ -1506,7 +1511,8 @@ def get_generated_json_files(tempdir=None, flight=False):
.skip_category('Go') # TODO(ARROW-7948): Decimal + Go
.skip_category('Rust'),
- generate_datetime_case(),
+ generate_datetime_case()
+ .skip_category('Rust'),
generate_interval_case()
.skip_category('JS') # TODO(ARROW-5239): Intervals + JS
@@ -1559,7 +1565,8 @@ def get_generated_json_files(tempdir=None, flight=False):
generate_nested_dictionary_case()
.skip_category('Go')
.skip_category('Java') # TODO(ARROW-7779)
- .skip_category('JS'),
+ .skip_category('JS')
+ .skip_category('Rust'),
generate_extension_case()
.skip_category('Go')
@@ -1569,7 +1576,8 @@ def get_generated_json_files(tempdir=None, flight=False):
if flight:
file_objs.append(generate_primitive_case([24 * 1024],
- name='large_batch'))
+ name='large_batch')
+ .skip_category('Rust'))
generated_paths = []
for file_obj in file_objs:
diff --git a/docker-compose.yml b/docker-compose.yml
index 9d838ef..f90596b 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1088,7 +1088,8 @@ services:
ARROW_CPP_EXE_PATH: /build/cpp/debug
# Running integration tests serially until ARROW-8176 resolved
command:
- ["/arrow/ci/scripts/cpp_build.sh /arrow /build &&
+ ["/arrow/ci/scripts/rust_build.sh /arrow /build &&
+ /arrow/ci/scripts/cpp_build.sh /arrow /build &&
/arrow/ci/scripts/go_build.sh /arrow &&
/arrow/ci/scripts/java_build.sh /arrow /build &&
/arrow/ci/scripts/js_build.sh /arrow /build &&
diff --git a/rust/integration-testing/src/bin/arrow-json-integration-test.rs b/rust/integration-testing/src/bin/arrow-json-integration-test.rs
index f562adf..a83e7ed 100644
--- a/rust/integration-testing/src/bin/arrow-json-integration-test.rs
+++ b/rust/integration-testing/src/bin/arrow-json-integration-test.rs
@@ -192,7 +192,13 @@ fn record_batch_from_json(
.zip(json_col.data.unwrap())
{
match is_valid {
- 1 => b.append_value(value.as_i64().unwrap()),
+ 1 => b.append_value(
+ value
+ .as_str()
+ .unwrap()
+ .parse()
+ .expect("Unable to parse string as i64"),
+ ),
_ => b.append_null(),
}
.unwrap();
@@ -261,7 +267,13 @@ fn record_batch_from_json(
.zip(json_col.data.unwrap())
{
match is_valid {
- 1 => b.append_value(value.as_u64().unwrap()),
+ 1 => b.append_value(
+ value
+ .as_str()
+ .unwrap()
+ .parse()
+ .expect("Unable to parse string as u64"),
+ ),
_ => b.append_null(),
}
.unwrap();
@@ -424,6 +436,12 @@ fn validate(arrow_name: &str, json_name: &str, verbose: bool) -> Result<()> {
)));
}
+ // compare number of batches
+ assert!(
+ json_batches.len() == arrow_reader.num_batches(),
+ "JSON batches and Arrow batches are unequal"
+ );
+
if verbose {
eprintln!(
"Schemas match. JSON file has {} batches.",