You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ag...@apache.org on 2021/04/18 16:21:46 UTC
[arrow-datafusion] branch master updated (0d5f00a -> bb110c3)
This is an automated email from the ASF dual-hosted git repository.
agrove pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git.
from 0d5f00a Removed c_glib.
new 479e1e2 Update Cargo.toml
new cb96853 remove arrow and parquet crates
new bb110c3 Update DataFusion dependencies on arrow-rs
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
rust/Cargo.toml | 8 +-
rust/arrow-flight/Cargo.toml | 45 -
rust/arrow-flight/README.md | 29 -
rust/arrow-flight/build.rs | 54 -
rust/arrow-flight/examples/server.rs | 131 -
rust/arrow-flight/src/arrow.flight.protocol.rs | 1039 -----
rust/arrow-flight/src/lib.rs | 20 -
rust/arrow-flight/src/utils.rs | 167 -
.../.cargo/config | 22 -
rust/arrow-pyarrow-integration-testing/.gitignore | 2 -
rust/arrow-pyarrow-integration-testing/Cargo.toml | 38 -
rust/arrow-pyarrow-integration-testing/README.md | 57 -
.../pyproject.toml | 20 -
rust/arrow-pyarrow-integration-testing/src/lib.rs | 188 -
.../tests/test_sql.py | 99 -
rust/arrow/Cargo.toml | 151 -
rust/arrow/README.md | 206 -
rust/arrow/benches/aggregate_kernels.rs | 67 -
rust/arrow/benches/arithmetic_kernels.rs | 103 -
rust/arrow/benches/array_from_vec.rs | 120 -
rust/arrow/benches/array_slice.rs | 52 -
rust/arrow/benches/bit_length_kernel.rs | 46 -
rust/arrow/benches/boolean_kernels.rs | 51 -
rust/arrow/benches/buffer_bit_ops.rs | 59 -
rust/arrow/benches/buffer_create.rs | 190 -
rust/arrow/benches/builder.rs | 116 -
rust/arrow/benches/cast_kernels.rs | 185 -
rust/arrow/benches/comparison_kernels.rs | 201 -
rust/arrow/benches/concatenate_kernel.rs | 66 -
rust/arrow/benches/csv_writer.rs | 70 -
rust/arrow/benches/equal.rs | 57 -
rust/arrow/benches/filter_kernels.rs | 106 -
rust/arrow/benches/json_reader.rs | 112 -
rust/arrow/benches/length_kernel.rs | 47 -
rust/arrow/benches/mutable_array.rs | 60 -
rust/arrow/benches/sort_kernel.rs | 121 -
rust/arrow/benches/take_kernels.rs | 128 -
rust/arrow/build.rs | 26 -
rust/arrow/examples/builders.rs | 131 -
rust/arrow/examples/dynamic_types.rs | 101 -
rust/arrow/examples/read_csv.rs | 43 -
rust/arrow/examples/read_csv_infer_schema.rs | 36 -
rust/arrow/examples/tensor_builder.rs | 67 -
rust/arrow/format-0ed34c83.patch | 220 -
rust/arrow/regen.sh | 157 -
rust/arrow/src/alloc/alignment.rs | 119 -
rust/arrow/src/alloc/mod.rs | 136 -
rust/arrow/src/alloc/types.rs | 71 -
rust/arrow/src/arch/avx512.rs | 73 -
rust/arrow/src/arch/mod.rs | 22 -
rust/arrow/src/array/array.rs | 640 ---
rust/arrow/src/array/array_binary.rs | 1157 -----
rust/arrow/src/array/array_boolean.rs | 291 --
rust/arrow/src/array/array_dictionary.rs | 408 --
rust/arrow/src/array/array_list.rs | 1056 -----
rust/arrow/src/array/array_primitive.rs | 942 ----
rust/arrow/src/array/array_string.rs | 528 ---
rust/arrow/src/array/array_struct.rs | 531 ---
rust/arrow/src/array/array_union.rs | 831 ----
rust/arrow/src/array/builder.rs | 3171 --------------
rust/arrow/src/array/cast.rs | 84 -
rust/arrow/src/array/data.rs | 679 ---
rust/arrow/src/array/equal/boolean.rs | 93 -
rust/arrow/src/array/equal/decimal.rs | 76 -
rust/arrow/src/array/equal/dictionary.rs | 82 -
rust/arrow/src/array/equal/fixed_binary.rs | 76 -
rust/arrow/src/array/equal/fixed_list.rs | 80 -
rust/arrow/src/array/equal/list.rs | 172 -
rust/arrow/src/array/equal/mod.rs | 1277 ------
rust/arrow/src/array/equal/null.rs | 31 -
rust/arrow/src/array/equal/primitive.rs | 73 -
rust/arrow/src/array/equal/structure.rs | 90 -
rust/arrow/src/array/equal/utils.rs | 264 --
rust/arrow/src/array/equal/variable_size.rs | 110 -
rust/arrow/src/array/equal_json.rs | 1113 -----
rust/arrow/src/array/ffi.rs | 168 -
rust/arrow/src/array/iterator.rs | 527 ---
rust/arrow/src/array/mod.rs | 283 --
rust/arrow/src/array/null.rs | 155 -
rust/arrow/src/array/ord.rs | 310 --
rust/arrow/src/array/raw_pointer.rs | 64 -
rust/arrow/src/array/transform/boolean.rs | 45 -
rust/arrow/src/array/transform/fixed_binary.rs | 65 -
rust/arrow/src/array/transform/list.rs | 99 -
rust/arrow/src/array/transform/mod.rs | 1206 -----
rust/arrow/src/array/transform/null.rs | 26 -
rust/arrow/src/array/transform/primitive.rs | 40 -
rust/arrow/src/array/transform/structure.rs | 67 -
rust/arrow/src/array/transform/utils.rs | 76 -
rust/arrow/src/array/transform/variable_size.rs | 105 -
rust/arrow/src/bitmap.rs | 157 -
rust/arrow/src/buffer/immutable.rs | 541 ---
rust/arrow/src/buffer/mod.rs | 69 -
rust/arrow/src/buffer/mutable.rs | 749 ----
rust/arrow/src/buffer/ops.rs | 429 --
rust/arrow/src/bytes.rs | 159 -
rust/arrow/src/compute/kernels/aggregate.rs | 975 -----
rust/arrow/src/compute/kernels/arithmetic.rs | 1009 -----
rust/arrow/src/compute/kernels/arity.rs | 74 -
rust/arrow/src/compute/kernels/boolean.rs | 1146 -----
rust/arrow/src/compute/kernels/cast.rs | 3843 ----------------
rust/arrow/src/compute/kernels/cast_utils.rs | 299 --
rust/arrow/src/compute/kernels/comparison.rs | 1619 -------
rust/arrow/src/compute/kernels/concat.rs | 387 --
rust/arrow/src/compute/kernels/filter.rs | 584 ---
rust/arrow/src/compute/kernels/length.rs | 385 --
rust/arrow/src/compute/kernels/limit.rs | 200 -
rust/arrow/src/compute/kernels/mod.rs | 37 -
rust/arrow/src/compute/kernels/regexp.rs | 160 -
rust/arrow/src/compute/kernels/sort.rs | 2246 ----------
rust/arrow/src/compute/kernels/substring.rs | 269 --
rust/arrow/src/compute/kernels/take.rs | 1621 -------
rust/arrow/src/compute/kernels/temporal.rs | 187 -
rust/arrow/src/compute/kernels/window.rs | 109 -
rust/arrow/src/compute/kernels/zip.rs | 87 -
rust/arrow/src/compute/mod.rs | 36 -
rust/arrow/src/compute/util.rs | 463 --
rust/arrow/src/csv/mod.rs | 27 -
rust/arrow/src/csv/reader.rs | 1291 ------
rust/arrow/src/csv/writer.rs | 651 ---
rust/arrow/src/datatypes/datatype.rs | 477 --
rust/arrow/src/datatypes/field.rs | 541 ---
rust/arrow/src/datatypes/mod.rs | 1241 ------
rust/arrow/src/datatypes/native.rs | 333 --
rust/arrow/src/datatypes/numeric.rs | 534 ---
rust/arrow/src/datatypes/schema.rs | 337 --
rust/arrow/src/datatypes/types.rs | 185 -
rust/arrow/src/error.rs | 134 -
rust/arrow/src/ffi.rs | 997 -----
rust/arrow/src/ipc/convert.rs | 871 ----
rust/arrow/src/ipc/gen/File.rs | 491 ---
rust/arrow/src/ipc/gen/Message.rs | 1346 ------
rust/arrow/src/ipc/gen/Schema.rs | 4586 --------------------
rust/arrow/src/ipc/gen/SparseTensor.rs | 1902 --------
rust/arrow/src/ipc/gen/Tensor.rs | 913 ----
rust/arrow/src/ipc/gen/mod.rs | 31 -
rust/arrow/src/ipc/mod.rs | 39 -
rust/arrow/src/ipc/reader.rs | 1160 -----
rust/arrow/src/ipc/writer.rs | 1160 -----
rust/arrow/src/json/mod.rs | 27 -
rust/arrow/src/json/reader.rs | 2949 -------------
rust/arrow/src/json/writer.rs | 1206 -----
rust/arrow/src/lib.rs | 162 -
rust/arrow/src/record_batch.rs | 434 --
rust/arrow/src/temporal_conversions.rs | 151 -
rust/arrow/src/tensor.rs | 495 ---
rust/arrow/src/util/bench_util.rs | 155 -
rust/arrow/src/util/bit_chunk_iterator.rs | 257 --
rust/arrow/src/util/bit_util.rs | 322 --
rust/arrow/src/util/data_gen.rs | 347 --
rust/arrow/src/util/display.rs | 298 --
rust/arrow/src/util/integration_util.rs | 957 ----
rust/arrow/src/util/mod.rs | 31 -
rust/arrow/src/util/pretty.rs | 421 --
rust/arrow/src/util/serialization.rs | 33 -
rust/arrow/src/util/string_writer.rs | 105 -
rust/arrow/src/util/test_util.rs | 211 -
rust/arrow/src/util/trusted_len.rs | 82 -
rust/arrow/src/zz_memory_check.rs | 31 -
rust/arrow/test/data/arrays.json | 3 -
rust/arrow/test/data/basic.json | 12 -
rust/arrow/test/data/basic_nulls.json | 12 -
rust/arrow/test/data/integration.json | 808 ----
rust/arrow/test/data/list_string_dict_nested.json | 3 -
.../test/data/list_string_dict_nested_nulls.json | 3 -
rust/arrow/test/data/mixed_arrays.json | 4 -
rust/arrow/test/data/mixed_arrays.json.gz | Bin 141 -> 0 bytes
rust/arrow/test/data/nested_structs.json | 4 -
rust/arrow/test/data/null_test.csv | 6 -
rust/arrow/test/data/uk_cities.csv | 37 -
rust/arrow/test/data/uk_cities_with_headers.csv | 38 -
rust/arrow/test/data/various_types.csv | 6 -
rust/arrow/test/data/various_types_invalid.csv | 6 -
rust/benchmarks/Cargo.toml | 4 +-
rust/datafusion-examples/Cargo.toml | 4 +-
rust/datafusion/Cargo.toml | 4 +-
rust/integration-testing/Cargo.toml | 45 -
rust/integration-testing/README.md | 30 -
.../src/bin/arrow-file-to-stream.rs | 43 -
.../src/bin/arrow-json-integration-test.rs | 180 -
.../src/bin/arrow-stream-to-file.rs | 34 -
.../src/bin/flight-test-integration-client.rs | 62 -
.../src/bin/flight-test-integration-server.rs | 55 -
.../src/flight_client_scenarios.rs | 20 -
.../flight_client_scenarios/auth_basic_proto.rs | 109 -
.../flight_client_scenarios/integration_test.rs | 271 --
.../src/flight_client_scenarios/middleware.rs | 83 -
.../src/flight_server_scenarios.rs | 48 -
.../flight_server_scenarios/auth_basic_proto.rs | 225 -
.../flight_server_scenarios/integration_test.rs | 385 --
.../src/flight_server_scenarios/middleware.rs | 150 -
rust/integration-testing/src/lib.rs | 601 ---
rust/parquet/Cargo.toml | 78 -
rust/parquet/README.md | 126 -
rust/parquet/benches/arrow_writer.rs | 202 -
rust/parquet/build.rs | 43 -
rust/parquet/src/arrow/array_reader.rs | 2530 -----------
rust/parquet/src/arrow/arrow_reader.rs | 671 ---
rust/parquet/src/arrow/arrow_writer.rs | 1402 ------
rust/parquet/src/arrow/converter.rs | 454 --
rust/parquet/src/arrow/levels.rs | 1411 ------
rust/parquet/src/arrow/mod.rs | 69 -
rust/parquet/src/arrow/record_reader.rs | 794 ----
rust/parquet/src/arrow/schema.rs | 1945 ---------
rust/parquet/src/basic.rs | 1969 ---------
rust/parquet/src/bin/parquet-read.rs | 128 -
rust/parquet/src/bin/parquet-rowcount.rs | 87 -
rust/parquet/src/bin/parquet-schema.rs | 104 -
rust/parquet/src/column/mod.rs | 124 -
rust/parquet/src/column/page.rs | 306 --
rust/parquet/src/column/reader.rs | 1356 ------
rust/parquet/src/column/writer.rs | 1908 --------
rust/parquet/src/compression.rs | 393 --
rust/parquet/src/data_type.rs | 1358 ------
rust/parquet/src/encodings/decoding.rs | 1387 ------
rust/parquet/src/encodings/encoding.rs | 1334 ------
rust/parquet/src/encodings/levels.rs | 563 ---
rust/parquet/src/encodings/mod.rs | 21 -
rust/parquet/src/encodings/rle.rs | 831 ----
rust/parquet/src/errors.rs | 146 -
rust/parquet/src/file/footer.rs | 263 --
rust/parquet/src/file/metadata.rs | 789 ----
rust/parquet/src/file/mod.rs | 110 -
rust/parquet/src/file/properties.rs | 679 ---
rust/parquet/src/file/reader.rs | 206 -
rust/parquet/src/file/serialized_reader.rs | 771 ----
rust/parquet/src/file/statistics.rs | 664 ---
rust/parquet/src/file/writer.rs | 1164 -----
rust/parquet/src/lib.rs | 57 -
rust/parquet/src/record/api.rs | 1846 --------
rust/parquet/src/record/mod.rs | 28 -
rust/parquet/src/record/reader.rs | 1667 -------
rust/parquet/src/record/record_writer.rs | 26 -
rust/parquet/src/record/triplet.rs | 561 ---
rust/parquet/src/schema/mod.rs | 67 -
rust/parquet/src/schema/parser.rs | 1241 ------
rust/parquet/src/schema/printer.rs | 827 ----
rust/parquet/src/schema/types.rs | 2078 ---------
rust/parquet/src/schema/visitor.rs | 240 -
rust/parquet/src/util/bit_packing.rs | 3662 ----------------
rust/parquet/src/util/bit_util.rs | 1143 -----
rust/parquet/src/util/cursor.rs | 260 --
rust/parquet/src/util/hash_util.rs | 172 -
rust/parquet/src/util/io.rs | 329 --
rust/parquet/src/util/memory.rs | 532 ---
rust/parquet/src/util/mod.rs | 27 -
rust/parquet/src/util/test_common/file_util.rs | 73 -
rust/parquet/src/util/test_common/mod.rs | 33 -
rust/parquet/src/util/test_common/page_util.rs | 313 --
rust/parquet/src/util/test_common/rand_gen.rs | 139 -
rust/parquet/tests/custom_writer.rs | 100 -
rust/parquet_derive/Cargo.toml | 42 -
rust/parquet_derive/README.md | 98 -
rust/parquet_derive/src/lib.rs | 126 -
rust/parquet_derive/src/parquet_field.rs | 920 ----
rust/parquet_derive_test/Cargo.toml | 32 -
rust/parquet_derive_test/src/lib.rs | 131 -
257 files changed, 7 insertions(+), 114731 deletions(-)
delete mode 100644 rust/arrow-flight/Cargo.toml
delete mode 100644 rust/arrow-flight/README.md
delete mode 100644 rust/arrow-flight/build.rs
delete mode 100644 rust/arrow-flight/examples/server.rs
delete mode 100644 rust/arrow-flight/src/arrow.flight.protocol.rs
delete mode 100644 rust/arrow-flight/src/lib.rs
delete mode 100644 rust/arrow-flight/src/utils.rs
delete mode 100644 rust/arrow-pyarrow-integration-testing/.cargo/config
delete mode 100644 rust/arrow-pyarrow-integration-testing/.gitignore
delete mode 100644 rust/arrow-pyarrow-integration-testing/Cargo.toml
delete mode 100644 rust/arrow-pyarrow-integration-testing/README.md
delete mode 100644 rust/arrow-pyarrow-integration-testing/pyproject.toml
delete mode 100644 rust/arrow-pyarrow-integration-testing/src/lib.rs
delete mode 100644 rust/arrow-pyarrow-integration-testing/tests/test_sql.py
delete mode 100644 rust/arrow/Cargo.toml
delete mode 100644 rust/arrow/README.md
delete mode 100644 rust/arrow/benches/aggregate_kernels.rs
delete mode 100644 rust/arrow/benches/arithmetic_kernels.rs
delete mode 100644 rust/arrow/benches/array_from_vec.rs
delete mode 100644 rust/arrow/benches/array_slice.rs
delete mode 100644 rust/arrow/benches/bit_length_kernel.rs
delete mode 100644 rust/arrow/benches/boolean_kernels.rs
delete mode 100644 rust/arrow/benches/buffer_bit_ops.rs
delete mode 100644 rust/arrow/benches/buffer_create.rs
delete mode 100644 rust/arrow/benches/builder.rs
delete mode 100644 rust/arrow/benches/cast_kernels.rs
delete mode 100644 rust/arrow/benches/comparison_kernels.rs
delete mode 100644 rust/arrow/benches/concatenate_kernel.rs
delete mode 100644 rust/arrow/benches/csv_writer.rs
delete mode 100644 rust/arrow/benches/equal.rs
delete mode 100644 rust/arrow/benches/filter_kernels.rs
delete mode 100644 rust/arrow/benches/json_reader.rs
delete mode 100644 rust/arrow/benches/length_kernel.rs
delete mode 100644 rust/arrow/benches/mutable_array.rs
delete mode 100644 rust/arrow/benches/sort_kernel.rs
delete mode 100644 rust/arrow/benches/take_kernels.rs
delete mode 100644 rust/arrow/build.rs
delete mode 100644 rust/arrow/examples/builders.rs
delete mode 100644 rust/arrow/examples/dynamic_types.rs
delete mode 100644 rust/arrow/examples/read_csv.rs
delete mode 100644 rust/arrow/examples/read_csv_infer_schema.rs
delete mode 100644 rust/arrow/examples/tensor_builder.rs
delete mode 100644 rust/arrow/format-0ed34c83.patch
delete mode 100755 rust/arrow/regen.sh
delete mode 100644 rust/arrow/src/alloc/alignment.rs
delete mode 100644 rust/arrow/src/alloc/mod.rs
delete mode 100644 rust/arrow/src/alloc/types.rs
delete mode 100644 rust/arrow/src/arch/avx512.rs
delete mode 100644 rust/arrow/src/arch/mod.rs
delete mode 100644 rust/arrow/src/array/array.rs
delete mode 100644 rust/arrow/src/array/array_binary.rs
delete mode 100644 rust/arrow/src/array/array_boolean.rs
delete mode 100644 rust/arrow/src/array/array_dictionary.rs
delete mode 100644 rust/arrow/src/array/array_list.rs
delete mode 100644 rust/arrow/src/array/array_primitive.rs
delete mode 100644 rust/arrow/src/array/array_string.rs
delete mode 100644 rust/arrow/src/array/array_struct.rs
delete mode 100644 rust/arrow/src/array/array_union.rs
delete mode 100644 rust/arrow/src/array/builder.rs
delete mode 100644 rust/arrow/src/array/cast.rs
delete mode 100644 rust/arrow/src/array/data.rs
delete mode 100644 rust/arrow/src/array/equal/boolean.rs
delete mode 100644 rust/arrow/src/array/equal/decimal.rs
delete mode 100644 rust/arrow/src/array/equal/dictionary.rs
delete mode 100644 rust/arrow/src/array/equal/fixed_binary.rs
delete mode 100644 rust/arrow/src/array/equal/fixed_list.rs
delete mode 100644 rust/arrow/src/array/equal/list.rs
delete mode 100644 rust/arrow/src/array/equal/mod.rs
delete mode 100644 rust/arrow/src/array/equal/null.rs
delete mode 100644 rust/arrow/src/array/equal/primitive.rs
delete mode 100644 rust/arrow/src/array/equal/structure.rs
delete mode 100644 rust/arrow/src/array/equal/utils.rs
delete mode 100644 rust/arrow/src/array/equal/variable_size.rs
delete mode 100644 rust/arrow/src/array/equal_json.rs
delete mode 100644 rust/arrow/src/array/ffi.rs
delete mode 100644 rust/arrow/src/array/iterator.rs
delete mode 100644 rust/arrow/src/array/mod.rs
delete mode 100644 rust/arrow/src/array/null.rs
delete mode 100644 rust/arrow/src/array/ord.rs
delete mode 100644 rust/arrow/src/array/raw_pointer.rs
delete mode 100644 rust/arrow/src/array/transform/boolean.rs
delete mode 100644 rust/arrow/src/array/transform/fixed_binary.rs
delete mode 100644 rust/arrow/src/array/transform/list.rs
delete mode 100644 rust/arrow/src/array/transform/mod.rs
delete mode 100644 rust/arrow/src/array/transform/null.rs
delete mode 100644 rust/arrow/src/array/transform/primitive.rs
delete mode 100644 rust/arrow/src/array/transform/structure.rs
delete mode 100644 rust/arrow/src/array/transform/utils.rs
delete mode 100644 rust/arrow/src/array/transform/variable_size.rs
delete mode 100644 rust/arrow/src/bitmap.rs
delete mode 100644 rust/arrow/src/buffer/immutable.rs
delete mode 100644 rust/arrow/src/buffer/mod.rs
delete mode 100644 rust/arrow/src/buffer/mutable.rs
delete mode 100644 rust/arrow/src/buffer/ops.rs
delete mode 100644 rust/arrow/src/bytes.rs
delete mode 100644 rust/arrow/src/compute/kernels/aggregate.rs
delete mode 100644 rust/arrow/src/compute/kernels/arithmetic.rs
delete mode 100644 rust/arrow/src/compute/kernels/arity.rs
delete mode 100644 rust/arrow/src/compute/kernels/boolean.rs
delete mode 100644 rust/arrow/src/compute/kernels/cast.rs
delete mode 100644 rust/arrow/src/compute/kernels/cast_utils.rs
delete mode 100644 rust/arrow/src/compute/kernels/comparison.rs
delete mode 100644 rust/arrow/src/compute/kernels/concat.rs
delete mode 100644 rust/arrow/src/compute/kernels/filter.rs
delete mode 100644 rust/arrow/src/compute/kernels/length.rs
delete mode 100644 rust/arrow/src/compute/kernels/limit.rs
delete mode 100644 rust/arrow/src/compute/kernels/mod.rs
delete mode 100644 rust/arrow/src/compute/kernels/regexp.rs
delete mode 100644 rust/arrow/src/compute/kernels/sort.rs
delete mode 100644 rust/arrow/src/compute/kernels/substring.rs
delete mode 100644 rust/arrow/src/compute/kernels/take.rs
delete mode 100644 rust/arrow/src/compute/kernels/temporal.rs
delete mode 100644 rust/arrow/src/compute/kernels/window.rs
delete mode 100644 rust/arrow/src/compute/kernels/zip.rs
delete mode 100644 rust/arrow/src/compute/mod.rs
delete mode 100644 rust/arrow/src/compute/util.rs
delete mode 100644 rust/arrow/src/csv/mod.rs
delete mode 100644 rust/arrow/src/csv/reader.rs
delete mode 100644 rust/arrow/src/csv/writer.rs
delete mode 100644 rust/arrow/src/datatypes/datatype.rs
delete mode 100644 rust/arrow/src/datatypes/field.rs
delete mode 100644 rust/arrow/src/datatypes/mod.rs
delete mode 100644 rust/arrow/src/datatypes/native.rs
delete mode 100644 rust/arrow/src/datatypes/numeric.rs
delete mode 100644 rust/arrow/src/datatypes/schema.rs
delete mode 100644 rust/arrow/src/datatypes/types.rs
delete mode 100644 rust/arrow/src/error.rs
delete mode 100644 rust/arrow/src/ffi.rs
delete mode 100644 rust/arrow/src/ipc/convert.rs
delete mode 100644 rust/arrow/src/ipc/gen/File.rs
delete mode 100644 rust/arrow/src/ipc/gen/Message.rs
delete mode 100644 rust/arrow/src/ipc/gen/Schema.rs
delete mode 100644 rust/arrow/src/ipc/gen/SparseTensor.rs
delete mode 100644 rust/arrow/src/ipc/gen/Tensor.rs
delete mode 100644 rust/arrow/src/ipc/gen/mod.rs
delete mode 100644 rust/arrow/src/ipc/mod.rs
delete mode 100644 rust/arrow/src/ipc/reader.rs
delete mode 100644 rust/arrow/src/ipc/writer.rs
delete mode 100644 rust/arrow/src/json/mod.rs
delete mode 100644 rust/arrow/src/json/reader.rs
delete mode 100644 rust/arrow/src/json/writer.rs
delete mode 100644 rust/arrow/src/lib.rs
delete mode 100644 rust/arrow/src/record_batch.rs
delete mode 100644 rust/arrow/src/temporal_conversions.rs
delete mode 100644 rust/arrow/src/tensor.rs
delete mode 100644 rust/arrow/src/util/bench_util.rs
delete mode 100644 rust/arrow/src/util/bit_chunk_iterator.rs
delete mode 100644 rust/arrow/src/util/bit_util.rs
delete mode 100644 rust/arrow/src/util/data_gen.rs
delete mode 100644 rust/arrow/src/util/display.rs
delete mode 100644 rust/arrow/src/util/integration_util.rs
delete mode 100644 rust/arrow/src/util/mod.rs
delete mode 100644 rust/arrow/src/util/pretty.rs
delete mode 100644 rust/arrow/src/util/serialization.rs
delete mode 100644 rust/arrow/src/util/string_writer.rs
delete mode 100644 rust/arrow/src/util/test_util.rs
delete mode 100644 rust/arrow/src/util/trusted_len.rs
delete mode 100644 rust/arrow/src/zz_memory_check.rs
delete mode 100644 rust/arrow/test/data/arrays.json
delete mode 100644 rust/arrow/test/data/basic.json
delete mode 100644 rust/arrow/test/data/basic_nulls.json
delete mode 100644 rust/arrow/test/data/integration.json
delete mode 100644 rust/arrow/test/data/list_string_dict_nested.json
delete mode 100644 rust/arrow/test/data/list_string_dict_nested_nulls.json
delete mode 100644 rust/arrow/test/data/mixed_arrays.json
delete mode 100644 rust/arrow/test/data/mixed_arrays.json.gz
delete mode 100644 rust/arrow/test/data/nested_structs.json
delete mode 100644 rust/arrow/test/data/null_test.csv
delete mode 100644 rust/arrow/test/data/uk_cities.csv
delete mode 100644 rust/arrow/test/data/uk_cities_with_headers.csv
delete mode 100644 rust/arrow/test/data/various_types.csv
delete mode 100644 rust/arrow/test/data/various_types_invalid.csv
delete mode 100644 rust/integration-testing/Cargo.toml
delete mode 100644 rust/integration-testing/README.md
delete mode 100644 rust/integration-testing/src/bin/arrow-file-to-stream.rs
delete mode 100644 rust/integration-testing/src/bin/arrow-json-integration-test.rs
delete mode 100644 rust/integration-testing/src/bin/arrow-stream-to-file.rs
delete mode 100644 rust/integration-testing/src/bin/flight-test-integration-client.rs
delete mode 100644 rust/integration-testing/src/bin/flight-test-integration-server.rs
delete mode 100644 rust/integration-testing/src/flight_client_scenarios.rs
delete mode 100644 rust/integration-testing/src/flight_client_scenarios/auth_basic_proto.rs
delete mode 100644 rust/integration-testing/src/flight_client_scenarios/integration_test.rs
delete mode 100644 rust/integration-testing/src/flight_client_scenarios/middleware.rs
delete mode 100644 rust/integration-testing/src/flight_server_scenarios.rs
delete mode 100644 rust/integration-testing/src/flight_server_scenarios/auth_basic_proto.rs
delete mode 100644 rust/integration-testing/src/flight_server_scenarios/integration_test.rs
delete mode 100644 rust/integration-testing/src/flight_server_scenarios/middleware.rs
delete mode 100644 rust/integration-testing/src/lib.rs
delete mode 100644 rust/parquet/Cargo.toml
delete mode 100644 rust/parquet/README.md
delete mode 100644 rust/parquet/benches/arrow_writer.rs
delete mode 100644 rust/parquet/build.rs
delete mode 100644 rust/parquet/src/arrow/array_reader.rs
delete mode 100644 rust/parquet/src/arrow/arrow_reader.rs
delete mode 100644 rust/parquet/src/arrow/arrow_writer.rs
delete mode 100644 rust/parquet/src/arrow/converter.rs
delete mode 100644 rust/parquet/src/arrow/levels.rs
delete mode 100644 rust/parquet/src/arrow/mod.rs
delete mode 100644 rust/parquet/src/arrow/record_reader.rs
delete mode 100644 rust/parquet/src/arrow/schema.rs
delete mode 100644 rust/parquet/src/basic.rs
delete mode 100644 rust/parquet/src/bin/parquet-read.rs
delete mode 100644 rust/parquet/src/bin/parquet-rowcount.rs
delete mode 100644 rust/parquet/src/bin/parquet-schema.rs
delete mode 100644 rust/parquet/src/column/mod.rs
delete mode 100644 rust/parquet/src/column/page.rs
delete mode 100644 rust/parquet/src/column/reader.rs
delete mode 100644 rust/parquet/src/column/writer.rs
delete mode 100644 rust/parquet/src/compression.rs
delete mode 100644 rust/parquet/src/data_type.rs
delete mode 100644 rust/parquet/src/encodings/decoding.rs
delete mode 100644 rust/parquet/src/encodings/encoding.rs
delete mode 100644 rust/parquet/src/encodings/levels.rs
delete mode 100644 rust/parquet/src/encodings/mod.rs
delete mode 100644 rust/parquet/src/encodings/rle.rs
delete mode 100644 rust/parquet/src/errors.rs
delete mode 100644 rust/parquet/src/file/footer.rs
delete mode 100644 rust/parquet/src/file/metadata.rs
delete mode 100644 rust/parquet/src/file/mod.rs
delete mode 100644 rust/parquet/src/file/properties.rs
delete mode 100644 rust/parquet/src/file/reader.rs
delete mode 100644 rust/parquet/src/file/serialized_reader.rs
delete mode 100644 rust/parquet/src/file/statistics.rs
delete mode 100644 rust/parquet/src/file/writer.rs
delete mode 100644 rust/parquet/src/lib.rs
delete mode 100644 rust/parquet/src/record/api.rs
delete mode 100644 rust/parquet/src/record/mod.rs
delete mode 100644 rust/parquet/src/record/reader.rs
delete mode 100644 rust/parquet/src/record/record_writer.rs
delete mode 100644 rust/parquet/src/record/triplet.rs
delete mode 100644 rust/parquet/src/schema/mod.rs
delete mode 100644 rust/parquet/src/schema/parser.rs
delete mode 100644 rust/parquet/src/schema/printer.rs
delete mode 100644 rust/parquet/src/schema/types.rs
delete mode 100644 rust/parquet/src/schema/visitor.rs
delete mode 100644 rust/parquet/src/util/bit_packing.rs
delete mode 100644 rust/parquet/src/util/bit_util.rs
delete mode 100644 rust/parquet/src/util/cursor.rs
delete mode 100644 rust/parquet/src/util/hash_util.rs
delete mode 100644 rust/parquet/src/util/io.rs
delete mode 100644 rust/parquet/src/util/memory.rs
delete mode 100644 rust/parquet/src/util/mod.rs
delete mode 100644 rust/parquet/src/util/test_common/file_util.rs
delete mode 100644 rust/parquet/src/util/test_common/mod.rs
delete mode 100644 rust/parquet/src/util/test_common/page_util.rs
delete mode 100644 rust/parquet/src/util/test_common/rand_gen.rs
delete mode 100644 rust/parquet/tests/custom_writer.rs
delete mode 100644 rust/parquet_derive/Cargo.toml
delete mode 100644 rust/parquet_derive/README.md
delete mode 100644 rust/parquet_derive/src/lib.rs
delete mode 100644 rust/parquet_derive/src/parquet_field.rs
delete mode 100644 rust/parquet_derive_test/Cargo.toml
delete mode 100644 rust/parquet_derive_test/src/lib.rs
[arrow-datafusion] 02/03: remove arrow and parquet crates
Posted by ag...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
commit cb968539f415f1c8d1a1a3c6c7e4281e9b643ad4
Author: Andy Grove <an...@gmail.com>
AuthorDate: Sun Apr 18 10:12:32 2021 -0600
remove arrow and parquet crates
---
rust/arrow-flight/Cargo.toml | 45 -
rust/arrow-flight/README.md | 29 -
rust/arrow-flight/build.rs | 54 -
rust/arrow-flight/examples/server.rs | 131 -
rust/arrow-flight/src/arrow.flight.protocol.rs | 1039 -----
rust/arrow-flight/src/lib.rs | 20 -
rust/arrow-flight/src/utils.rs | 167 -
.../.cargo/config | 22 -
rust/arrow-pyarrow-integration-testing/.gitignore | 2 -
rust/arrow-pyarrow-integration-testing/Cargo.toml | 38 -
rust/arrow-pyarrow-integration-testing/README.md | 57 -
.../pyproject.toml | 20 -
rust/arrow-pyarrow-integration-testing/src/lib.rs | 188 -
.../tests/test_sql.py | 99 -
rust/arrow/Cargo.toml | 151 -
rust/arrow/README.md | 206 -
rust/arrow/benches/aggregate_kernels.rs | 67 -
rust/arrow/benches/arithmetic_kernels.rs | 103 -
rust/arrow/benches/array_from_vec.rs | 120 -
rust/arrow/benches/array_slice.rs | 52 -
rust/arrow/benches/bit_length_kernel.rs | 46 -
rust/arrow/benches/boolean_kernels.rs | 51 -
rust/arrow/benches/buffer_bit_ops.rs | 59 -
rust/arrow/benches/buffer_create.rs | 190 -
rust/arrow/benches/builder.rs | 116 -
rust/arrow/benches/cast_kernels.rs | 185 -
rust/arrow/benches/comparison_kernels.rs | 201 -
rust/arrow/benches/concatenate_kernel.rs | 66 -
rust/arrow/benches/csv_writer.rs | 70 -
rust/arrow/benches/equal.rs | 57 -
rust/arrow/benches/filter_kernels.rs | 106 -
rust/arrow/benches/json_reader.rs | 112 -
rust/arrow/benches/length_kernel.rs | 47 -
rust/arrow/benches/mutable_array.rs | 60 -
rust/arrow/benches/sort_kernel.rs | 121 -
rust/arrow/benches/take_kernels.rs | 128 -
rust/arrow/build.rs | 26 -
rust/arrow/examples/builders.rs | 131 -
rust/arrow/examples/dynamic_types.rs | 101 -
rust/arrow/examples/read_csv.rs | 43 -
rust/arrow/examples/read_csv_infer_schema.rs | 36 -
rust/arrow/examples/tensor_builder.rs | 67 -
rust/arrow/format-0ed34c83.patch | 220 -
rust/arrow/regen.sh | 157 -
rust/arrow/src/alloc/alignment.rs | 119 -
rust/arrow/src/alloc/mod.rs | 136 -
rust/arrow/src/alloc/types.rs | 71 -
rust/arrow/src/arch/avx512.rs | 73 -
rust/arrow/src/arch/mod.rs | 22 -
rust/arrow/src/array/array.rs | 640 ---
rust/arrow/src/array/array_binary.rs | 1157 -----
rust/arrow/src/array/array_boolean.rs | 291 --
rust/arrow/src/array/array_dictionary.rs | 408 --
rust/arrow/src/array/array_list.rs | 1056 -----
rust/arrow/src/array/array_primitive.rs | 942 ----
rust/arrow/src/array/array_string.rs | 528 ---
rust/arrow/src/array/array_struct.rs | 531 ---
rust/arrow/src/array/array_union.rs | 831 ----
rust/arrow/src/array/builder.rs | 3171 --------------
rust/arrow/src/array/cast.rs | 84 -
rust/arrow/src/array/data.rs | 679 ---
rust/arrow/src/array/equal/boolean.rs | 93 -
rust/arrow/src/array/equal/decimal.rs | 76 -
rust/arrow/src/array/equal/dictionary.rs | 82 -
rust/arrow/src/array/equal/fixed_binary.rs | 76 -
rust/arrow/src/array/equal/fixed_list.rs | 80 -
rust/arrow/src/array/equal/list.rs | 172 -
rust/arrow/src/array/equal/mod.rs | 1277 ------
rust/arrow/src/array/equal/null.rs | 31 -
rust/arrow/src/array/equal/primitive.rs | 73 -
rust/arrow/src/array/equal/structure.rs | 90 -
rust/arrow/src/array/equal/utils.rs | 264 --
rust/arrow/src/array/equal/variable_size.rs | 110 -
rust/arrow/src/array/equal_json.rs | 1113 -----
rust/arrow/src/array/ffi.rs | 168 -
rust/arrow/src/array/iterator.rs | 527 ---
rust/arrow/src/array/mod.rs | 283 --
rust/arrow/src/array/null.rs | 155 -
rust/arrow/src/array/ord.rs | 310 --
rust/arrow/src/array/raw_pointer.rs | 64 -
rust/arrow/src/array/transform/boolean.rs | 45 -
rust/arrow/src/array/transform/fixed_binary.rs | 65 -
rust/arrow/src/array/transform/list.rs | 99 -
rust/arrow/src/array/transform/mod.rs | 1206 -----
rust/arrow/src/array/transform/null.rs | 26 -
rust/arrow/src/array/transform/primitive.rs | 40 -
rust/arrow/src/array/transform/structure.rs | 67 -
rust/arrow/src/array/transform/utils.rs | 76 -
rust/arrow/src/array/transform/variable_size.rs | 105 -
rust/arrow/src/bitmap.rs | 157 -
rust/arrow/src/buffer/immutable.rs | 541 ---
rust/arrow/src/buffer/mod.rs | 69 -
rust/arrow/src/buffer/mutable.rs | 749 ----
rust/arrow/src/buffer/ops.rs | 429 --
rust/arrow/src/bytes.rs | 159 -
rust/arrow/src/compute/kernels/aggregate.rs | 975 -----
rust/arrow/src/compute/kernels/arithmetic.rs | 1009 -----
rust/arrow/src/compute/kernels/arity.rs | 74 -
rust/arrow/src/compute/kernels/boolean.rs | 1146 -----
rust/arrow/src/compute/kernels/cast.rs | 3843 ----------------
rust/arrow/src/compute/kernels/cast_utils.rs | 299 --
rust/arrow/src/compute/kernels/comparison.rs | 1619 -------
rust/arrow/src/compute/kernels/concat.rs | 387 --
rust/arrow/src/compute/kernels/filter.rs | 584 ---
rust/arrow/src/compute/kernels/length.rs | 385 --
rust/arrow/src/compute/kernels/limit.rs | 200 -
rust/arrow/src/compute/kernels/mod.rs | 37 -
rust/arrow/src/compute/kernels/regexp.rs | 160 -
rust/arrow/src/compute/kernels/sort.rs | 2246 ----------
rust/arrow/src/compute/kernels/substring.rs | 269 --
rust/arrow/src/compute/kernels/take.rs | 1621 -------
rust/arrow/src/compute/kernels/temporal.rs | 187 -
rust/arrow/src/compute/kernels/window.rs | 109 -
rust/arrow/src/compute/kernels/zip.rs | 87 -
rust/arrow/src/compute/mod.rs | 36 -
rust/arrow/src/compute/util.rs | 463 --
rust/arrow/src/csv/mod.rs | 27 -
rust/arrow/src/csv/reader.rs | 1291 ------
rust/arrow/src/csv/writer.rs | 651 ---
rust/arrow/src/datatypes/datatype.rs | 477 --
rust/arrow/src/datatypes/field.rs | 541 ---
rust/arrow/src/datatypes/mod.rs | 1241 ------
rust/arrow/src/datatypes/native.rs | 333 --
rust/arrow/src/datatypes/numeric.rs | 534 ---
rust/arrow/src/datatypes/schema.rs | 337 --
rust/arrow/src/datatypes/types.rs | 185 -
rust/arrow/src/error.rs | 134 -
rust/arrow/src/ffi.rs | 997 -----
rust/arrow/src/ipc/convert.rs | 871 ----
rust/arrow/src/ipc/gen/File.rs | 491 ---
rust/arrow/src/ipc/gen/Message.rs | 1346 ------
rust/arrow/src/ipc/gen/Schema.rs | 4586 --------------------
rust/arrow/src/ipc/gen/SparseTensor.rs | 1902 --------
rust/arrow/src/ipc/gen/Tensor.rs | 913 ----
rust/arrow/src/ipc/gen/mod.rs | 31 -
rust/arrow/src/ipc/mod.rs | 39 -
rust/arrow/src/ipc/reader.rs | 1160 -----
rust/arrow/src/ipc/writer.rs | 1160 -----
rust/arrow/src/json/mod.rs | 27 -
rust/arrow/src/json/reader.rs | 2949 -------------
rust/arrow/src/json/writer.rs | 1206 -----
rust/arrow/src/lib.rs | 162 -
rust/arrow/src/record_batch.rs | 434 --
rust/arrow/src/temporal_conversions.rs | 151 -
rust/arrow/src/tensor.rs | 495 ---
rust/arrow/src/util/bench_util.rs | 155 -
rust/arrow/src/util/bit_chunk_iterator.rs | 257 --
rust/arrow/src/util/bit_util.rs | 322 --
rust/arrow/src/util/data_gen.rs | 347 --
rust/arrow/src/util/display.rs | 298 --
rust/arrow/src/util/integration_util.rs | 957 ----
rust/arrow/src/util/mod.rs | 31 -
rust/arrow/src/util/pretty.rs | 421 --
rust/arrow/src/util/serialization.rs | 33 -
rust/arrow/src/util/string_writer.rs | 105 -
rust/arrow/src/util/test_util.rs | 211 -
rust/arrow/src/util/trusted_len.rs | 82 -
rust/arrow/src/zz_memory_check.rs | 31 -
rust/arrow/test/data/arrays.json | 3 -
rust/arrow/test/data/basic.json | 12 -
rust/arrow/test/data/basic_nulls.json | 12 -
rust/arrow/test/data/integration.json | 808 ----
rust/arrow/test/data/list_string_dict_nested.json | 3 -
.../test/data/list_string_dict_nested_nulls.json | 3 -
rust/arrow/test/data/mixed_arrays.json | 4 -
rust/arrow/test/data/mixed_arrays.json.gz | Bin 141 -> 0 bytes
rust/arrow/test/data/nested_structs.json | 4 -
rust/arrow/test/data/null_test.csv | 6 -
rust/arrow/test/data/uk_cities.csv | 37 -
rust/arrow/test/data/uk_cities_with_headers.csv | 38 -
rust/arrow/test/data/various_types.csv | 6 -
rust/arrow/test/data/various_types_invalid.csv | 6 -
rust/integration-testing/Cargo.toml | 45 -
rust/integration-testing/README.md | 30 -
.../src/bin/arrow-file-to-stream.rs | 43 -
.../src/bin/arrow-json-integration-test.rs | 180 -
.../src/bin/arrow-stream-to-file.rs | 34 -
.../src/bin/flight-test-integration-client.rs | 62 -
.../src/bin/flight-test-integration-server.rs | 55 -
.../src/flight_client_scenarios.rs | 20 -
.../flight_client_scenarios/auth_basic_proto.rs | 109 -
.../flight_client_scenarios/integration_test.rs | 271 --
.../src/flight_client_scenarios/middleware.rs | 83 -
.../src/flight_server_scenarios.rs | 48 -
.../flight_server_scenarios/auth_basic_proto.rs | 225 -
.../flight_server_scenarios/integration_test.rs | 385 --
.../src/flight_server_scenarios/middleware.rs | 150 -
rust/integration-testing/src/lib.rs | 601 ---
rust/parquet/Cargo.toml | 78 -
rust/parquet/README.md | 126 -
rust/parquet/benches/arrow_writer.rs | 202 -
rust/parquet/build.rs | 43 -
rust/parquet/src/arrow/array_reader.rs | 2530 -----------
rust/parquet/src/arrow/arrow_reader.rs | 671 ---
rust/parquet/src/arrow/arrow_writer.rs | 1402 ------
rust/parquet/src/arrow/converter.rs | 454 --
rust/parquet/src/arrow/levels.rs | 1411 ------
rust/parquet/src/arrow/mod.rs | 69 -
rust/parquet/src/arrow/record_reader.rs | 794 ----
rust/parquet/src/arrow/schema.rs | 1945 ---------
rust/parquet/src/basic.rs | 1969 ---------
rust/parquet/src/bin/parquet-read.rs | 128 -
rust/parquet/src/bin/parquet-rowcount.rs | 87 -
rust/parquet/src/bin/parquet-schema.rs | 104 -
rust/parquet/src/column/mod.rs | 124 -
rust/parquet/src/column/page.rs | 306 --
rust/parquet/src/column/reader.rs | 1356 ------
rust/parquet/src/column/writer.rs | 1908 --------
rust/parquet/src/compression.rs | 393 --
rust/parquet/src/data_type.rs | 1358 ------
rust/parquet/src/encodings/decoding.rs | 1387 ------
rust/parquet/src/encodings/encoding.rs | 1334 ------
rust/parquet/src/encodings/levels.rs | 563 ---
rust/parquet/src/encodings/mod.rs | 21 -
rust/parquet/src/encodings/rle.rs | 831 ----
rust/parquet/src/errors.rs | 146 -
rust/parquet/src/file/footer.rs | 263 --
rust/parquet/src/file/metadata.rs | 789 ----
rust/parquet/src/file/mod.rs | 110 -
rust/parquet/src/file/properties.rs | 679 ---
rust/parquet/src/file/reader.rs | 206 -
rust/parquet/src/file/serialized_reader.rs | 771 ----
rust/parquet/src/file/statistics.rs | 664 ---
rust/parquet/src/file/writer.rs | 1164 -----
rust/parquet/src/lib.rs | 57 -
rust/parquet/src/record/api.rs | 1846 --------
rust/parquet/src/record/mod.rs | 28 -
rust/parquet/src/record/reader.rs | 1667 -------
rust/parquet/src/record/record_writer.rs | 26 -
rust/parquet/src/record/triplet.rs | 561 ---
rust/parquet/src/schema/mod.rs | 67 -
rust/parquet/src/schema/parser.rs | 1241 ------
rust/parquet/src/schema/printer.rs | 827 ----
rust/parquet/src/schema/types.rs | 2078 ---------
rust/parquet/src/schema/visitor.rs | 240 -
rust/parquet/src/util/bit_packing.rs | 3662 ----------------
rust/parquet/src/util/bit_util.rs | 1143 -----
rust/parquet/src/util/cursor.rs | 260 --
rust/parquet/src/util/hash_util.rs | 172 -
rust/parquet/src/util/io.rs | 329 --
rust/parquet/src/util/memory.rs | 532 ---
rust/parquet/src/util/mod.rs | 27 -
rust/parquet/src/util/test_common/file_util.rs | 73 -
rust/parquet/src/util/test_common/mod.rs | 33 -
rust/parquet/src/util/test_common/page_util.rs | 313 --
rust/parquet/src/util/test_common/rand_gen.rs | 139 -
rust/parquet/tests/custom_writer.rs | 100 -
rust/parquet_derive/Cargo.toml | 42 -
rust/parquet_derive/README.md | 98 -
rust/parquet_derive/src/lib.rs | 126 -
rust/parquet_derive/src/parquet_field.rs | 920 ----
rust/parquet_derive_test/Cargo.toml | 32 -
rust/parquet_derive_test/src/lib.rs | 131 -
253 files changed, 114718 deletions(-)
diff --git a/rust/arrow-flight/Cargo.toml b/rust/arrow-flight/Cargo.toml
deleted file mode 100644
index c607b56..0000000
--- a/rust/arrow-flight/Cargo.toml
+++ /dev/null
@@ -1,45 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-[package]
-name = "arrow-flight"
-description = "Apache Arrow Flight"
-version = "4.0.0-SNAPSHOT"
-edition = "2018"
-authors = ["Apache Arrow <de...@arrow.apache.org>"]
-homepage = "https://github.com/apache/arrow"
-repository = "https://github.com/apache/arrow"
-license = "Apache-2.0"
-
-[dependencies]
-arrow = { path = "../arrow", version = "4.0.0-SNAPSHOT" }
-tonic = "0.4"
-bytes = "1"
-prost = "0.7"
-prost-derive = "0.7"
-tokio = { version = "1.0", features = ["macros", "rt", "rt-multi-thread"] }
-futures = { version = "0.3", default-features = false, features = ["alloc"]}
-
-[build-dependencies]
-tonic-build = "0.4"
-# Pin specific version of the tonic-build dependencies to avoid auto-generated
-# (and checked in) arrow.flight.protocol.rs from changing
-proc-macro2 = "=1.0.24"
-
-#[lib]
-#name = "flight"
-#path = "src/lib.rs"
diff --git a/rust/arrow-flight/README.md b/rust/arrow-flight/README.md
deleted file mode 100644
index ba63f65..0000000
--- a/rust/arrow-flight/README.md
+++ /dev/null
@@ -1,29 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-# Apache Arrow Flight
-
-Apache Arrow Flight is a gRPC based protocol for exchanging Arrow data between processes. See the blog post [Introducing Apache Arrow Flight: A Framework for Fast Data Transport](https://arrow.apache.org/blog/2019/10/13/introducing-arrow-flight/) for more information.
-
-This crate simply provides the Rust implementation of the [Flight.proto](../../format/Flight.proto) gRPC protocol and provides an example that demonstrates how to build a Flight server implemented with Tonic.
-
-Note that building a Flight server also requires an implementation of Arrow IPC which is based on the Flatbuffers serialization framework. The Rust implementation of Arrow IPC is not yet complete although the generated Flatbuffers code is available as part of the core Arrow crate.
-
-
-
diff --git a/rust/arrow-flight/build.rs b/rust/arrow-flight/build.rs
deleted file mode 100644
index ca23255..0000000
--- a/rust/arrow-flight/build.rs
+++ /dev/null
@@ -1,54 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::{
- env,
- fs::OpenOptions,
- io::{Read, Write},
- path::Path,
-};
-
-fn main() -> Result<(), Box<dyn std::error::Error>> {
- // avoid rerunning build if the file has not changed
- println!("cargo:rerun-if-changed=../../format/Flight.proto");
-
- // override the build location, in order to check in the changes to proto files
- env::set_var("OUT_DIR", "src");
-
- // The current working directory can vary depending on how the project is being
- // built or released so we build an absolute path to the proto file
- let path = Path::new("../../format/Flight.proto");
- if path.exists() {
- tonic_build::compile_protos("../../format/Flight.proto")?;
- // read file contents to string
- let mut file = OpenOptions::new()
- .read(true)
- .open("src/arrow.flight.protocol.rs")?;
- let mut buffer = String::new();
- file.read_to_string(&mut buffer)?;
- // append warning that file was auto-generate
- let mut file = OpenOptions::new()
- .write(true)
- .truncate(true)
- .open("src/arrow.flight.protocol.rs")?;
- file.write_all("// This file was automatically generated through the build.rs script, and should not be edited.\n\n".as_bytes())?;
- file.write_all(buffer.as_bytes())?;
- }
-
- // As the proto file is checked in, the build should not fail if the file is not found
- Ok(())
-}
diff --git a/rust/arrow-flight/examples/server.rs b/rust/arrow-flight/examples/server.rs
deleted file mode 100644
index 75d0537..0000000
--- a/rust/arrow-flight/examples/server.rs
+++ /dev/null
@@ -1,131 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::pin::Pin;
-
-use futures::Stream;
-use tonic::transport::Server;
-use tonic::{Request, Response, Status, Streaming};
-
-use arrow_flight::{
- flight_service_server::FlightService, flight_service_server::FlightServiceServer,
- Action, ActionType, Criteria, Empty, FlightData, FlightDescriptor, FlightInfo,
- HandshakeRequest, HandshakeResponse, PutResult, SchemaResult, Ticket,
-};
-
-#[derive(Clone)]
-pub struct FlightServiceImpl {}
-
-#[tonic::async_trait]
-impl FlightService for FlightServiceImpl {
- type HandshakeStream = Pin<
- Box<dyn Stream<Item = Result<HandshakeResponse, Status>> + Send + Sync + 'static>,
- >;
- type ListFlightsStream =
- Pin<Box<dyn Stream<Item = Result<FlightInfo, Status>> + Send + Sync + 'static>>;
- type DoGetStream =
- Pin<Box<dyn Stream<Item = Result<FlightData, Status>> + Send + Sync + 'static>>;
- type DoPutStream =
- Pin<Box<dyn Stream<Item = Result<PutResult, Status>> + Send + Sync + 'static>>;
- type DoActionStream = Pin<
- Box<
- dyn Stream<Item = Result<arrow_flight::Result, Status>>
- + Send
- + Sync
- + 'static,
- >,
- >;
- type ListActionsStream =
- Pin<Box<dyn Stream<Item = Result<ActionType, Status>> + Send + Sync + 'static>>;
- type DoExchangeStream =
- Pin<Box<dyn Stream<Item = Result<FlightData, Status>> + Send + Sync + 'static>>;
-
- async fn handshake(
- &self,
- _request: Request<Streaming<HandshakeRequest>>,
- ) -> Result<Response<Self::HandshakeStream>, Status> {
- Err(Status::unimplemented("Not yet implemented"))
- }
-
- async fn list_flights(
- &self,
- _request: Request<Criteria>,
- ) -> Result<Response<Self::ListFlightsStream>, Status> {
- Err(Status::unimplemented("Not yet implemented"))
- }
-
- async fn get_flight_info(
- &self,
- _request: Request<FlightDescriptor>,
- ) -> Result<Response<FlightInfo>, Status> {
- Err(Status::unimplemented("Not yet implemented"))
- }
-
- async fn get_schema(
- &self,
- _request: Request<FlightDescriptor>,
- ) -> Result<Response<SchemaResult>, Status> {
- Err(Status::unimplemented("Not yet implemented"))
- }
-
- async fn do_get(
- &self,
- _request: Request<Ticket>,
- ) -> Result<Response<Self::DoGetStream>, Status> {
- Err(Status::unimplemented("Not yet implemented"))
- }
-
- async fn do_put(
- &self,
- _request: Request<Streaming<FlightData>>,
- ) -> Result<Response<Self::DoPutStream>, Status> {
- Err(Status::unimplemented("Not yet implemented"))
- }
-
- async fn do_action(
- &self,
- _request: Request<Action>,
- ) -> Result<Response<Self::DoActionStream>, Status> {
- Err(Status::unimplemented("Not yet implemented"))
- }
-
- async fn list_actions(
- &self,
- _request: Request<Empty>,
- ) -> Result<Response<Self::ListActionsStream>, Status> {
- Err(Status::unimplemented("Not yet implemented"))
- }
-
- async fn do_exchange(
- &self,
- _request: Request<Streaming<FlightData>>,
- ) -> Result<Response<Self::DoExchangeStream>, Status> {
- Err(Status::unimplemented("Not yet implemented"))
- }
-}
-
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
- let addr = "[::1]:50051".parse()?;
- let service = FlightServiceImpl {};
-
- let svc = FlightServiceServer::new(service);
-
- Server::builder().add_service(svc).serve(addr).await?;
-
- Ok(())
-}
diff --git a/rust/arrow-flight/src/arrow.flight.protocol.rs b/rust/arrow-flight/src/arrow.flight.protocol.rs
deleted file mode 100644
index 5fce526..0000000
--- a/rust/arrow-flight/src/arrow.flight.protocol.rs
+++ /dev/null
@@ -1,1039 +0,0 @@
-// This file was automatically generated through the build.rs script, and should not be edited.
-
-///
-/// The request that a client provides to a server on handshake.
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct HandshakeRequest {
- ///
- /// A defined protocol version
- #[prost(uint64, tag = "1")]
- pub protocol_version: u64,
- ///
- /// Arbitrary auth/handshake info.
- #[prost(bytes = "vec", tag = "2")]
- pub payload: ::prost::alloc::vec::Vec<u8>,
-}
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct HandshakeResponse {
- ///
- /// A defined protocol version
- #[prost(uint64, tag = "1")]
- pub protocol_version: u64,
- ///
- /// Arbitrary auth/handshake info.
- #[prost(bytes = "vec", tag = "2")]
- pub payload: ::prost::alloc::vec::Vec<u8>,
-}
-///
-/// A message for doing simple auth.
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct BasicAuth {
- #[prost(string, tag = "2")]
- pub username: ::prost::alloc::string::String,
- #[prost(string, tag = "3")]
- pub password: ::prost::alloc::string::String,
-}
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct Empty {}
-///
-/// Describes an available action, including both the name used for execution
-/// along with a short description of the purpose of the action.
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct ActionType {
- #[prost(string, tag = "1")]
- pub r#type: ::prost::alloc::string::String,
- #[prost(string, tag = "2")]
- pub description: ::prost::alloc::string::String,
-}
-///
-/// A service specific expression that can be used to return a limited set
-/// of available Arrow Flight streams.
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct Criteria {
- #[prost(bytes = "vec", tag = "1")]
- pub expression: ::prost::alloc::vec::Vec<u8>,
-}
-///
-/// An opaque action specific for the service.
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct Action {
- #[prost(string, tag = "1")]
- pub r#type: ::prost::alloc::string::String,
- #[prost(bytes = "vec", tag = "2")]
- pub body: ::prost::alloc::vec::Vec<u8>,
-}
-///
-/// An opaque result returned after executing an action.
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct Result {
- #[prost(bytes = "vec", tag = "1")]
- pub body: ::prost::alloc::vec::Vec<u8>,
-}
-///
-/// Wrap the result of a getSchema call
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct SchemaResult {
- /// schema of the dataset as described in Schema.fbs::Schema.
- #[prost(bytes = "vec", tag = "1")]
- pub schema: ::prost::alloc::vec::Vec<u8>,
-}
-///
-/// The name or tag for a Flight. May be used as a way to retrieve or generate
-/// a flight or be used to expose a set of previously defined flights.
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct FlightDescriptor {
- #[prost(enumeration = "flight_descriptor::DescriptorType", tag = "1")]
- pub r#type: i32,
- ///
- /// Opaque value used to express a command. Should only be defined when
- /// type = CMD.
- #[prost(bytes = "vec", tag = "2")]
- pub cmd: ::prost::alloc::vec::Vec<u8>,
- ///
- /// List of strings identifying a particular dataset. Should only be defined
- /// when type = PATH.
- #[prost(string, repeated, tag = "3")]
- pub path: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
-}
-/// Nested message and enum types in `FlightDescriptor`.
-pub mod flight_descriptor {
- ///
- /// Describes what type of descriptor is defined.
- #[derive(
- Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration,
- )]
- #[repr(i32)]
- pub enum DescriptorType {
- /// Protobuf pattern, not used.
- Unknown = 0,
- ///
- /// A named path that identifies a dataset. A path is composed of a string
- /// or list of strings describing a particular dataset. This is conceptually
- /// similar to a path inside a filesystem.
- Path = 1,
- ///
- /// An opaque command to generate a dataset.
- Cmd = 2,
- }
-}
-///
-/// The access coordinates for retrieval of a dataset. With a FlightInfo, a
-/// consumer is able to determine how to retrieve a dataset.
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct FlightInfo {
- /// schema of the dataset as described in Schema.fbs::Schema.
- #[prost(bytes = "vec", tag = "1")]
- pub schema: ::prost::alloc::vec::Vec<u8>,
- ///
- /// The descriptor associated with this info.
- #[prost(message, optional, tag = "2")]
- pub flight_descriptor: ::core::option::Option<FlightDescriptor>,
- ///
- /// A list of endpoints associated with the flight. To consume the whole
- /// flight, all endpoints must be consumed.
- #[prost(message, repeated, tag = "3")]
- pub endpoint: ::prost::alloc::vec::Vec<FlightEndpoint>,
- /// Set these to -1 if unknown.
- #[prost(int64, tag = "4")]
- pub total_records: i64,
- #[prost(int64, tag = "5")]
- pub total_bytes: i64,
-}
-///
-/// A particular stream or split associated with a flight.
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct FlightEndpoint {
- ///
- /// Token used to retrieve this stream.
- #[prost(message, optional, tag = "1")]
- pub ticket: ::core::option::Option<Ticket>,
- ///
- /// A list of URIs where this ticket can be redeemed. If the list is
- /// empty, the expectation is that the ticket can only be redeemed on the
- /// current service where the ticket was generated.
- #[prost(message, repeated, tag = "2")]
- pub location: ::prost::alloc::vec::Vec<Location>,
-}
-///
-/// A location where a Flight service will accept retrieval of a particular
-/// stream given a ticket.
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct Location {
- #[prost(string, tag = "1")]
- pub uri: ::prost::alloc::string::String,
-}
-///
-/// An opaque identifier that the service can use to retrieve a particular
-/// portion of a stream.
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct Ticket {
- #[prost(bytes = "vec", tag = "1")]
- pub ticket: ::prost::alloc::vec::Vec<u8>,
-}
-///
-/// A batch of Arrow data as part of a stream of batches.
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct FlightData {
- ///
- /// The descriptor of the data. This is only relevant when a client is
- /// starting a new DoPut stream.
- #[prost(message, optional, tag = "1")]
- pub flight_descriptor: ::core::option::Option<FlightDescriptor>,
- ///
- /// Header for message data as described in Message.fbs::Message.
- #[prost(bytes = "vec", tag = "2")]
- pub data_header: ::prost::alloc::vec::Vec<u8>,
- ///
- /// Application-defined metadata.
- #[prost(bytes = "vec", tag = "3")]
- pub app_metadata: ::prost::alloc::vec::Vec<u8>,
- ///
- /// The actual batch of Arrow data. Preferably handled with minimal-copies
- /// coming last in the definition to help with sidecar patterns (it is
- /// expected that some implementations will fetch this field off the wire
- /// with specialized code to avoid extra memory copies).
- #[prost(bytes = "vec", tag = "1000")]
- pub data_body: ::prost::alloc::vec::Vec<u8>,
-}
-///*
-/// The response message associated with the submission of a DoPut.
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct PutResult {
- #[prost(bytes = "vec", tag = "1")]
- pub app_metadata: ::prost::alloc::vec::Vec<u8>,
-}
-#[doc = r" Generated client implementations."]
-pub mod flight_service_client {
- #![allow(unused_variables, dead_code, missing_docs)]
- use tonic::codegen::*;
- #[doc = ""]
- #[doc = " A flight service is an endpoint for retrieving or storing Arrow data. A"]
- #[doc = " flight service can expose one or more predefined endpoints that can be"]
- #[doc = " accessed using the Arrow Flight Protocol. Additionally, a flight service"]
- #[doc = " can expose a set of actions that are available."]
- pub struct FlightServiceClient<T> {
- inner: tonic::client::Grpc<T>,
- }
- impl FlightServiceClient<tonic::transport::Channel> {
- #[doc = r" Attempt to create a new client by connecting to a given endpoint."]
- pub async fn connect<D>(dst: D) -> Result<Self, tonic::transport::Error>
- where
- D: std::convert::TryInto<tonic::transport::Endpoint>,
- D::Error: Into<StdError>,
- {
- let conn = tonic::transport::Endpoint::new(dst)?.connect().await?;
- Ok(Self::new(conn))
- }
- }
- impl<T> FlightServiceClient<T>
- where
- T: tonic::client::GrpcService<tonic::body::BoxBody>,
- T::ResponseBody: Body + HttpBody + Send + 'static,
- T::Error: Into<StdError>,
- <T::ResponseBody as HttpBody>::Error: Into<StdError> + Send,
- {
- pub fn new(inner: T) -> Self {
- let inner = tonic::client::Grpc::new(inner);
- Self { inner }
- }
- pub fn with_interceptor(
- inner: T,
- interceptor: impl Into<tonic::Interceptor>,
- ) -> Self {
- let inner = tonic::client::Grpc::with_interceptor(inner, interceptor);
- Self { inner }
- }
- #[doc = ""]
- #[doc = " Handshake between client and server. Depending on the server, the"]
- #[doc = " handshake may be required to determine the token that should be used for"]
- #[doc = " future operations. Both request and response are streams to allow multiple"]
- #[doc = " round-trips depending on auth mechanism."]
- pub async fn handshake(
- &mut self,
- request: impl tonic::IntoStreamingRequest<Message = super::HandshakeRequest>,
- ) -> Result<
- tonic::Response<tonic::codec::Streaming<super::HandshakeResponse>>,
- tonic::Status,
- > {
- self.inner.ready().await.map_err(|e| {
- tonic::Status::new(
- tonic::Code::Unknown,
- format!("Service was not ready: {}", e.into()),
- )
- })?;
- let codec = tonic::codec::ProstCodec::default();
- let path = http::uri::PathAndQuery::from_static(
- "/arrow.flight.protocol.FlightService/Handshake",
- );
- self.inner
- .streaming(request.into_streaming_request(), path, codec)
- .await
- }
- #[doc = ""]
- #[doc = " Get a list of available streams given a particular criteria. Most flight"]
- #[doc = " services will expose one or more streams that are readily available for"]
- #[doc = " retrieval. This api allows listing the streams available for"]
- #[doc = " consumption. A user can also provide a criteria. The criteria can limit"]
- #[doc = " the subset of streams that can be listed via this interface. Each flight"]
- #[doc = " service allows its own definition of how to consume criteria."]
- pub async fn list_flights(
- &mut self,
- request: impl tonic::IntoRequest<super::Criteria>,
- ) -> Result<
- tonic::Response<tonic::codec::Streaming<super::FlightInfo>>,
- tonic::Status,
- > {
- self.inner.ready().await.map_err(|e| {
- tonic::Status::new(
- tonic::Code::Unknown,
- format!("Service was not ready: {}", e.into()),
- )
- })?;
- let codec = tonic::codec::ProstCodec::default();
- let path = http::uri::PathAndQuery::from_static(
- "/arrow.flight.protocol.FlightService/ListFlights",
- );
- self.inner
- .server_streaming(request.into_request(), path, codec)
- .await
- }
- #[doc = ""]
- #[doc = " For a given FlightDescriptor, get information about how the flight can be"]
- #[doc = " consumed. This is a useful interface if the consumer of the interface"]
- #[doc = " already can identify the specific flight to consume. This interface can"]
- #[doc = " also allow a consumer to generate a flight stream through a specified"]
- #[doc = " descriptor. For example, a flight descriptor might be something that"]
- #[doc = " includes a SQL statement or a Pickled Python operation that will be"]
- #[doc = " executed. In those cases, the descriptor will not be previously available"]
- #[doc = " within the list of available streams provided by ListFlights but will be"]
- #[doc = " available for consumption for the duration defined by the specific flight"]
- #[doc = " service."]
- pub async fn get_flight_info(
- &mut self,
- request: impl tonic::IntoRequest<super::FlightDescriptor>,
- ) -> Result<tonic::Response<super::FlightInfo>, tonic::Status> {
- self.inner.ready().await.map_err(|e| {
- tonic::Status::new(
- tonic::Code::Unknown,
- format!("Service was not ready: {}", e.into()),
- )
- })?;
- let codec = tonic::codec::ProstCodec::default();
- let path = http::uri::PathAndQuery::from_static(
- "/arrow.flight.protocol.FlightService/GetFlightInfo",
- );
- self.inner.unary(request.into_request(), path, codec).await
- }
- #[doc = ""]
- #[doc = " For a given FlightDescriptor, get the Schema as described in Schema.fbs::Schema"]
- #[doc = " This is used when a consumer needs the Schema of flight stream. Similar to"]
- #[doc = " GetFlightInfo this interface may generate a new flight that was not previously"]
- #[doc = " available in ListFlights."]
- pub async fn get_schema(
- &mut self,
- request: impl tonic::IntoRequest<super::FlightDescriptor>,
- ) -> Result<tonic::Response<super::SchemaResult>, tonic::Status> {
- self.inner.ready().await.map_err(|e| {
- tonic::Status::new(
- tonic::Code::Unknown,
- format!("Service was not ready: {}", e.into()),
- )
- })?;
- let codec = tonic::codec::ProstCodec::default();
- let path = http::uri::PathAndQuery::from_static(
- "/arrow.flight.protocol.FlightService/GetSchema",
- );
- self.inner.unary(request.into_request(), path, codec).await
- }
- #[doc = ""]
- #[doc = " Retrieve a single stream associated with a particular descriptor"]
- #[doc = " associated with the referenced ticket. A Flight can be composed of one or"]
- #[doc = " more streams where each stream can be retrieved using a separate opaque"]
- #[doc = " ticket that the flight service uses for managing a collection of streams."]
- pub async fn do_get(
- &mut self,
- request: impl tonic::IntoRequest<super::Ticket>,
- ) -> Result<
- tonic::Response<tonic::codec::Streaming<super::FlightData>>,
- tonic::Status,
- > {
- self.inner.ready().await.map_err(|e| {
- tonic::Status::new(
- tonic::Code::Unknown,
- format!("Service was not ready: {}", e.into()),
- )
- })?;
- let codec = tonic::codec::ProstCodec::default();
- let path = http::uri::PathAndQuery::from_static(
- "/arrow.flight.protocol.FlightService/DoGet",
- );
- self.inner
- .server_streaming(request.into_request(), path, codec)
- .await
- }
- #[doc = ""]
- #[doc = " Push a stream to the flight service associated with a particular"]
- #[doc = " flight stream. This allows a client of a flight service to upload a stream"]
- #[doc = " of data. Depending on the particular flight service, a client consumer"]
- #[doc = " could be allowed to upload a single stream per descriptor or an unlimited"]
- #[doc = " number. In the latter, the service might implement a 'seal' action that"]
- #[doc = " can be applied to a descriptor once all streams are uploaded."]
- pub async fn do_put(
- &mut self,
- request: impl tonic::IntoStreamingRequest<Message = super::FlightData>,
- ) -> Result<
- tonic::Response<tonic::codec::Streaming<super::PutResult>>,
- tonic::Status,
- > {
- self.inner.ready().await.map_err(|e| {
- tonic::Status::new(
- tonic::Code::Unknown,
- format!("Service was not ready: {}", e.into()),
- )
- })?;
- let codec = tonic::codec::ProstCodec::default();
- let path = http::uri::PathAndQuery::from_static(
- "/arrow.flight.protocol.FlightService/DoPut",
- );
- self.inner
- .streaming(request.into_streaming_request(), path, codec)
- .await
- }
- #[doc = ""]
- #[doc = " Open a bidirectional data channel for a given descriptor. This"]
- #[doc = " allows clients to send and receive arbitrary Arrow data and"]
- #[doc = " application-specific metadata in a single logical stream. In"]
- #[doc = " contrast to DoGet/DoPut, this is more suited for clients"]
- #[doc = " offloading computation (rather than storage) to a Flight service."]
- pub async fn do_exchange(
- &mut self,
- request: impl tonic::IntoStreamingRequest<Message = super::FlightData>,
- ) -> Result<
- tonic::Response<tonic::codec::Streaming<super::FlightData>>,
- tonic::Status,
- > {
- self.inner.ready().await.map_err(|e| {
- tonic::Status::new(
- tonic::Code::Unknown,
- format!("Service was not ready: {}", e.into()),
- )
- })?;
- let codec = tonic::codec::ProstCodec::default();
- let path = http::uri::PathAndQuery::from_static(
- "/arrow.flight.protocol.FlightService/DoExchange",
- );
- self.inner
- .streaming(request.into_streaming_request(), path, codec)
- .await
- }
- #[doc = ""]
- #[doc = " Flight services can support an arbitrary number of simple actions in"]
- #[doc = " addition to the possible ListFlights, GetFlightInfo, DoGet, DoPut"]
- #[doc = " operations that are potentially available. DoAction allows a flight client"]
- #[doc = " to do a specific action against a flight service. An action includes"]
- #[doc = " opaque request and response objects that are specific to the type action"]
- #[doc = " being undertaken."]
- pub async fn do_action(
- &mut self,
- request: impl tonic::IntoRequest<super::Action>,
- ) -> Result<tonic::Response<tonic::codec::Streaming<super::Result>>, tonic::Status>
- {
- self.inner.ready().await.map_err(|e| {
- tonic::Status::new(
- tonic::Code::Unknown,
- format!("Service was not ready: {}", e.into()),
- )
- })?;
- let codec = tonic::codec::ProstCodec::default();
- let path = http::uri::PathAndQuery::from_static(
- "/arrow.flight.protocol.FlightService/DoAction",
- );
- self.inner
- .server_streaming(request.into_request(), path, codec)
- .await
- }
- #[doc = ""]
- #[doc = " A flight service exposes all of the available action types that it has"]
- #[doc = " along with descriptions. This allows different flight consumers to"]
- #[doc = " understand the capabilities of the flight service."]
- pub async fn list_actions(
- &mut self,
- request: impl tonic::IntoRequest<super::Empty>,
- ) -> Result<
- tonic::Response<tonic::codec::Streaming<super::ActionType>>,
- tonic::Status,
- > {
- self.inner.ready().await.map_err(|e| {
- tonic::Status::new(
- tonic::Code::Unknown,
- format!("Service was not ready: {}", e.into()),
- )
- })?;
- let codec = tonic::codec::ProstCodec::default();
- let path = http::uri::PathAndQuery::from_static(
- "/arrow.flight.protocol.FlightService/ListActions",
- );
- self.inner
- .server_streaming(request.into_request(), path, codec)
- .await
- }
- }
- impl<T: Clone> Clone for FlightServiceClient<T> {
- fn clone(&self) -> Self {
- Self {
- inner: self.inner.clone(),
- }
- }
- }
- impl<T> std::fmt::Debug for FlightServiceClient<T> {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- write!(f, "FlightServiceClient {{ ... }}")
- }
- }
-}
-#[doc = r" Generated server implementations."]
-pub mod flight_service_server {
- #![allow(unused_variables, dead_code, missing_docs)]
- use tonic::codegen::*;
- #[doc = "Generated trait containing gRPC methods that should be implemented for use with FlightServiceServer."]
- #[async_trait]
- pub trait FlightService: Send + Sync + 'static {
- #[doc = "Server streaming response type for the Handshake method."]
- type HandshakeStream: futures_core::Stream<Item = Result<super::HandshakeResponse, tonic::Status>>
- + Send
- + Sync
- + 'static;
- #[doc = ""]
- #[doc = " Handshake between client and server. Depending on the server, the"]
- #[doc = " handshake may be required to determine the token that should be used for"]
- #[doc = " future operations. Both request and response are streams to allow multiple"]
- #[doc = " round-trips depending on auth mechanism."]
- async fn handshake(
- &self,
- request: tonic::Request<tonic::Streaming<super::HandshakeRequest>>,
- ) -> Result<tonic::Response<Self::HandshakeStream>, tonic::Status>;
- #[doc = "Server streaming response type for the ListFlights method."]
- type ListFlightsStream: futures_core::Stream<Item = Result<super::FlightInfo, tonic::Status>>
- + Send
- + Sync
- + 'static;
- #[doc = ""]
- #[doc = " Get a list of available streams given a particular criteria. Most flight"]
- #[doc = " services will expose one or more streams that are readily available for"]
- #[doc = " retrieval. This api allows listing the streams available for"]
- #[doc = " consumption. A user can also provide a criteria. The criteria can limit"]
- #[doc = " the subset of streams that can be listed via this interface. Each flight"]
- #[doc = " service allows its own definition of how to consume criteria."]
- async fn list_flights(
- &self,
- request: tonic::Request<super::Criteria>,
- ) -> Result<tonic::Response<Self::ListFlightsStream>, tonic::Status>;
- #[doc = ""]
- #[doc = " For a given FlightDescriptor, get information about how the flight can be"]
- #[doc = " consumed. This is a useful interface if the consumer of the interface"]
- #[doc = " already can identify the specific flight to consume. This interface can"]
- #[doc = " also allow a consumer to generate a flight stream through a specified"]
- #[doc = " descriptor. For example, a flight descriptor might be something that"]
- #[doc = " includes a SQL statement or a Pickled Python operation that will be"]
- #[doc = " executed. In those cases, the descriptor will not be previously available"]
- #[doc = " within the list of available streams provided by ListFlights but will be"]
- #[doc = " available for consumption for the duration defined by the specific flight"]
- #[doc = " service."]
- async fn get_flight_info(
- &self,
- request: tonic::Request<super::FlightDescriptor>,
- ) -> Result<tonic::Response<super::FlightInfo>, tonic::Status>;
- #[doc = ""]
- #[doc = " For a given FlightDescriptor, get the Schema as described in Schema.fbs::Schema"]
- #[doc = " This is used when a consumer needs the Schema of flight stream. Similar to"]
- #[doc = " GetFlightInfo this interface may generate a new flight that was not previously"]
- #[doc = " available in ListFlights."]
- async fn get_schema(
- &self,
- request: tonic::Request<super::FlightDescriptor>,
- ) -> Result<tonic::Response<super::SchemaResult>, tonic::Status>;
- #[doc = "Server streaming response type for the DoGet method."]
- type DoGetStream: futures_core::Stream<Item = Result<super::FlightData, tonic::Status>>
- + Send
- + Sync
- + 'static;
- #[doc = ""]
- #[doc = " Retrieve a single stream associated with a particular descriptor"]
- #[doc = " associated with the referenced ticket. A Flight can be composed of one or"]
- #[doc = " more streams where each stream can be retrieved using a separate opaque"]
- #[doc = " ticket that the flight service uses for managing a collection of streams."]
- async fn do_get(
- &self,
- request: tonic::Request<super::Ticket>,
- ) -> Result<tonic::Response<Self::DoGetStream>, tonic::Status>;
- #[doc = "Server streaming response type for the DoPut method."]
- type DoPutStream: futures_core::Stream<Item = Result<super::PutResult, tonic::Status>>
- + Send
- + Sync
- + 'static;
- #[doc = ""]
- #[doc = " Push a stream to the flight service associated with a particular"]
- #[doc = " flight stream. This allows a client of a flight service to upload a stream"]
- #[doc = " of data. Depending on the particular flight service, a client consumer"]
- #[doc = " could be allowed to upload a single stream per descriptor or an unlimited"]
- #[doc = " number. In the latter, the service might implement a 'seal' action that"]
- #[doc = " can be applied to a descriptor once all streams are uploaded."]
- async fn do_put(
- &self,
- request: tonic::Request<tonic::Streaming<super::FlightData>>,
- ) -> Result<tonic::Response<Self::DoPutStream>, tonic::Status>;
- #[doc = "Server streaming response type for the DoExchange method."]
- type DoExchangeStream: futures_core::Stream<Item = Result<super::FlightData, tonic::Status>>
- + Send
- + Sync
- + 'static;
- #[doc = ""]
- #[doc = " Open a bidirectional data channel for a given descriptor. This"]
- #[doc = " allows clients to send and receive arbitrary Arrow data and"]
- #[doc = " application-specific metadata in a single logical stream. In"]
- #[doc = " contrast to DoGet/DoPut, this is more suited for clients"]
- #[doc = " offloading computation (rather than storage) to a Flight service."]
- async fn do_exchange(
- &self,
- request: tonic::Request<tonic::Streaming<super::FlightData>>,
- ) -> Result<tonic::Response<Self::DoExchangeStream>, tonic::Status>;
- #[doc = "Server streaming response type for the DoAction method."]
- type DoActionStream: futures_core::Stream<Item = Result<super::Result, tonic::Status>>
- + Send
- + Sync
- + 'static;
- #[doc = ""]
- #[doc = " Flight services can support an arbitrary number of simple actions in"]
- #[doc = " addition to the possible ListFlights, GetFlightInfo, DoGet, DoPut"]
- #[doc = " operations that are potentially available. DoAction allows a flight client"]
- #[doc = " to do a specific action against a flight service. An action includes"]
- #[doc = " opaque request and response objects that are specific to the type action"]
- #[doc = " being undertaken."]
- async fn do_action(
- &self,
- request: tonic::Request<super::Action>,
- ) -> Result<tonic::Response<Self::DoActionStream>, tonic::Status>;
- #[doc = "Server streaming response type for the ListActions method."]
- type ListActionsStream: futures_core::Stream<Item = Result<super::ActionType, tonic::Status>>
- + Send
- + Sync
- + 'static;
- #[doc = ""]
- #[doc = " A flight service exposes all of the available action types that it has"]
- #[doc = " along with descriptions. This allows different flight consumers to"]
- #[doc = " understand the capabilities of the flight service."]
- async fn list_actions(
- &self,
- request: tonic::Request<super::Empty>,
- ) -> Result<tonic::Response<Self::ListActionsStream>, tonic::Status>;
- }
- #[doc = ""]
- #[doc = " A flight service is an endpoint for retrieving or storing Arrow data. A"]
- #[doc = " flight service can expose one or more predefined endpoints that can be"]
- #[doc = " accessed using the Arrow Flight Protocol. Additionally, a flight service"]
- #[doc = " can expose a set of actions that are available."]
- #[derive(Debug)]
- pub struct FlightServiceServer<T: FlightService> {
- inner: _Inner<T>,
- }
- struct _Inner<T>(Arc<T>, Option<tonic::Interceptor>);
- impl<T: FlightService> FlightServiceServer<T> {
- pub fn new(inner: T) -> Self {
- let inner = Arc::new(inner);
- let inner = _Inner(inner, None);
- Self { inner }
- }
- pub fn with_interceptor(
- inner: T,
- interceptor: impl Into<tonic::Interceptor>,
- ) -> Self {
- let inner = Arc::new(inner);
- let inner = _Inner(inner, Some(interceptor.into()));
- Self { inner }
- }
- }
- impl<T, B> Service<http::Request<B>> for FlightServiceServer<T>
- where
- T: FlightService,
- B: HttpBody + Send + Sync + 'static,
- B::Error: Into<StdError> + Send + 'static,
- {
- type Response = http::Response<tonic::body::BoxBody>;
- type Error = Never;
- type Future = BoxFuture<Self::Response, Self::Error>;
- fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
- Poll::Ready(Ok(()))
- }
- fn call(&mut self, req: http::Request<B>) -> Self::Future {
- let inner = self.inner.clone();
- match req.uri().path() {
- "/arrow.flight.protocol.FlightService/Handshake" => {
- #[allow(non_camel_case_types)]
- struct HandshakeSvc<T: FlightService>(pub Arc<T>);
- impl<T: FlightService>
- tonic::server::StreamingService<super::HandshakeRequest>
- for HandshakeSvc<T>
- {
- type Response = super::HandshakeResponse;
- type ResponseStream = T::HandshakeStream;
- type Future = BoxFuture<
- tonic::Response<Self::ResponseStream>,
- tonic::Status,
- >;
- fn call(
- &mut self,
- request: tonic::Request<
- tonic::Streaming<super::HandshakeRequest>,
- >,
- ) -> Self::Future {
- let inner = self.0.clone();
- let fut = async move { (*inner).handshake(request).await };
- Box::pin(fut)
- }
- }
- let inner = self.inner.clone();
- let fut = async move {
- let interceptor = inner.1;
- let inner = inner.0;
- let method = HandshakeSvc(inner);
- let codec = tonic::codec::ProstCodec::default();
- let mut grpc = if let Some(interceptor) = interceptor {
- tonic::server::Grpc::with_interceptor(codec, interceptor)
- } else {
- tonic::server::Grpc::new(codec)
- };
- let res = grpc.streaming(method, req).await;
- Ok(res)
- };
- Box::pin(fut)
- }
- "/arrow.flight.protocol.FlightService/ListFlights" => {
- #[allow(non_camel_case_types)]
- struct ListFlightsSvc<T: FlightService>(pub Arc<T>);
- impl<T: FlightService>
- tonic::server::ServerStreamingService<super::Criteria>
- for ListFlightsSvc<T>
- {
- type Response = super::FlightInfo;
- type ResponseStream = T::ListFlightsStream;
- type Future = BoxFuture<
- tonic::Response<Self::ResponseStream>,
- tonic::Status,
- >;
- fn call(
- &mut self,
- request: tonic::Request<super::Criteria>,
- ) -> Self::Future {
- let inner = self.0.clone();
- let fut = async move { (*inner).list_flights(request).await };
- Box::pin(fut)
- }
- }
- let inner = self.inner.clone();
- let fut = async move {
- let interceptor = inner.1;
- let inner = inner.0;
- let method = ListFlightsSvc(inner);
- let codec = tonic::codec::ProstCodec::default();
- let mut grpc = if let Some(interceptor) = interceptor {
- tonic::server::Grpc::with_interceptor(codec, interceptor)
- } else {
- tonic::server::Grpc::new(codec)
- };
- let res = grpc.server_streaming(method, req).await;
- Ok(res)
- };
- Box::pin(fut)
- }
- "/arrow.flight.protocol.FlightService/GetFlightInfo" => {
- #[allow(non_camel_case_types)]
- struct GetFlightInfoSvc<T: FlightService>(pub Arc<T>);
- impl<T: FlightService>
- tonic::server::UnaryService<super::FlightDescriptor>
- for GetFlightInfoSvc<T>
- {
- type Response = super::FlightInfo;
- type Future =
- BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
- fn call(
- &mut self,
- request: tonic::Request<super::FlightDescriptor>,
- ) -> Self::Future {
- let inner = self.0.clone();
- let fut =
- async move { (*inner).get_flight_info(request).await };
- Box::pin(fut)
- }
- }
- let inner = self.inner.clone();
- let fut = async move {
- let interceptor = inner.1.clone();
- let inner = inner.0;
- let method = GetFlightInfoSvc(inner);
- let codec = tonic::codec::ProstCodec::default();
- let mut grpc = if let Some(interceptor) = interceptor {
- tonic::server::Grpc::with_interceptor(codec, interceptor)
- } else {
- tonic::server::Grpc::new(codec)
- };
- let res = grpc.unary(method, req).await;
- Ok(res)
- };
- Box::pin(fut)
- }
- "/arrow.flight.protocol.FlightService/GetSchema" => {
- #[allow(non_camel_case_types)]
- struct GetSchemaSvc<T: FlightService>(pub Arc<T>);
- impl<T: FlightService>
- tonic::server::UnaryService<super::FlightDescriptor>
- for GetSchemaSvc<T>
- {
- type Response = super::SchemaResult;
- type Future =
- BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
- fn call(
- &mut self,
- request: tonic::Request<super::FlightDescriptor>,
- ) -> Self::Future {
- let inner = self.0.clone();
- let fut = async move { (*inner).get_schema(request).await };
- Box::pin(fut)
- }
- }
- let inner = self.inner.clone();
- let fut = async move {
- let interceptor = inner.1.clone();
- let inner = inner.0;
- let method = GetSchemaSvc(inner);
- let codec = tonic::codec::ProstCodec::default();
- let mut grpc = if let Some(interceptor) = interceptor {
- tonic::server::Grpc::with_interceptor(codec, interceptor)
- } else {
- tonic::server::Grpc::new(codec)
- };
- let res = grpc.unary(method, req).await;
- Ok(res)
- };
- Box::pin(fut)
- }
- "/arrow.flight.protocol.FlightService/DoGet" => {
- #[allow(non_camel_case_types)]
- struct DoGetSvc<T: FlightService>(pub Arc<T>);
- impl<T: FlightService>
- tonic::server::ServerStreamingService<super::Ticket>
- for DoGetSvc<T>
- {
- type Response = super::FlightData;
- type ResponseStream = T::DoGetStream;
- type Future = BoxFuture<
- tonic::Response<Self::ResponseStream>,
- tonic::Status,
- >;
- fn call(
- &mut self,
- request: tonic::Request<super::Ticket>,
- ) -> Self::Future {
- let inner = self.0.clone();
- let fut = async move { (*inner).do_get(request).await };
- Box::pin(fut)
- }
- }
- let inner = self.inner.clone();
- let fut = async move {
- let interceptor = inner.1;
- let inner = inner.0;
- let method = DoGetSvc(inner);
- let codec = tonic::codec::ProstCodec::default();
- let mut grpc = if let Some(interceptor) = interceptor {
- tonic::server::Grpc::with_interceptor(codec, interceptor)
- } else {
- tonic::server::Grpc::new(codec)
- };
- let res = grpc.server_streaming(method, req).await;
- Ok(res)
- };
- Box::pin(fut)
- }
- "/arrow.flight.protocol.FlightService/DoPut" => {
- #[allow(non_camel_case_types)]
- struct DoPutSvc<T: FlightService>(pub Arc<T>);
- impl<T: FlightService>
- tonic::server::StreamingService<super::FlightData>
- for DoPutSvc<T>
- {
- type Response = super::PutResult;
- type ResponseStream = T::DoPutStream;
- type Future = BoxFuture<
- tonic::Response<Self::ResponseStream>,
- tonic::Status,
- >;
- fn call(
- &mut self,
- request: tonic::Request<tonic::Streaming<super::FlightData>>,
- ) -> Self::Future {
- let inner = self.0.clone();
- let fut = async move { (*inner).do_put(request).await };
- Box::pin(fut)
- }
- }
- let inner = self.inner.clone();
- let fut = async move {
- let interceptor = inner.1;
- let inner = inner.0;
- let method = DoPutSvc(inner);
- let codec = tonic::codec::ProstCodec::default();
- let mut grpc = if let Some(interceptor) = interceptor {
- tonic::server::Grpc::with_interceptor(codec, interceptor)
- } else {
- tonic::server::Grpc::new(codec)
- };
- let res = grpc.streaming(method, req).await;
- Ok(res)
- };
- Box::pin(fut)
- }
- "/arrow.flight.protocol.FlightService/DoExchange" => {
- #[allow(non_camel_case_types)]
- struct DoExchangeSvc<T: FlightService>(pub Arc<T>);
- impl<T: FlightService>
- tonic::server::StreamingService<super::FlightData>
- for DoExchangeSvc<T>
- {
- type Response = super::FlightData;
- type ResponseStream = T::DoExchangeStream;
- type Future = BoxFuture<
- tonic::Response<Self::ResponseStream>,
- tonic::Status,
- >;
- fn call(
- &mut self,
- request: tonic::Request<tonic::Streaming<super::FlightData>>,
- ) -> Self::Future {
- let inner = self.0.clone();
- let fut = async move { (*inner).do_exchange(request).await };
- Box::pin(fut)
- }
- }
- let inner = self.inner.clone();
- let fut = async move {
- let interceptor = inner.1;
- let inner = inner.0;
- let method = DoExchangeSvc(inner);
- let codec = tonic::codec::ProstCodec::default();
- let mut grpc = if let Some(interceptor) = interceptor {
- tonic::server::Grpc::with_interceptor(codec, interceptor)
- } else {
- tonic::server::Grpc::new(codec)
- };
- let res = grpc.streaming(method, req).await;
- Ok(res)
- };
- Box::pin(fut)
- }
- "/arrow.flight.protocol.FlightService/DoAction" => {
- #[allow(non_camel_case_types)]
- struct DoActionSvc<T: FlightService>(pub Arc<T>);
- impl<T: FlightService>
- tonic::server::ServerStreamingService<super::Action>
- for DoActionSvc<T>
- {
- type Response = super::Result;
- type ResponseStream = T::DoActionStream;
- type Future = BoxFuture<
- tonic::Response<Self::ResponseStream>,
- tonic::Status,
- >;
- fn call(
- &mut self,
- request: tonic::Request<super::Action>,
- ) -> Self::Future {
- let inner = self.0.clone();
- let fut = async move { (*inner).do_action(request).await };
- Box::pin(fut)
- }
- }
- let inner = self.inner.clone();
- let fut = async move {
- let interceptor = inner.1;
- let inner = inner.0;
- let method = DoActionSvc(inner);
- let codec = tonic::codec::ProstCodec::default();
- let mut grpc = if let Some(interceptor) = interceptor {
- tonic::server::Grpc::with_interceptor(codec, interceptor)
- } else {
- tonic::server::Grpc::new(codec)
- };
- let res = grpc.server_streaming(method, req).await;
- Ok(res)
- };
- Box::pin(fut)
- }
- "/arrow.flight.protocol.FlightService/ListActions" => {
- #[allow(non_camel_case_types)]
- struct ListActionsSvc<T: FlightService>(pub Arc<T>);
- impl<T: FlightService>
- tonic::server::ServerStreamingService<super::Empty>
- for ListActionsSvc<T>
- {
- type Response = super::ActionType;
- type ResponseStream = T::ListActionsStream;
- type Future = BoxFuture<
- tonic::Response<Self::ResponseStream>,
- tonic::Status,
- >;
- fn call(
- &mut self,
- request: tonic::Request<super::Empty>,
- ) -> Self::Future {
- let inner = self.0.clone();
- let fut = async move { (*inner).list_actions(request).await };
- Box::pin(fut)
- }
- }
- let inner = self.inner.clone();
- let fut = async move {
- let interceptor = inner.1;
- let inner = inner.0;
- let method = ListActionsSvc(inner);
- let codec = tonic::codec::ProstCodec::default();
- let mut grpc = if let Some(interceptor) = interceptor {
- tonic::server::Grpc::with_interceptor(codec, interceptor)
- } else {
- tonic::server::Grpc::new(codec)
- };
- let res = grpc.server_streaming(method, req).await;
- Ok(res)
- };
- Box::pin(fut)
- }
- _ => Box::pin(async move {
- Ok(http::Response::builder()
- .status(200)
- .header("grpc-status", "12")
- .header("content-type", "application/grpc")
- .body(tonic::body::BoxBody::empty())
- .unwrap())
- }),
- }
- }
- }
- impl<T: FlightService> Clone for FlightServiceServer<T> {
- fn clone(&self) -> Self {
- let inner = self.inner.clone();
- Self { inner }
- }
- }
- impl<T: FlightService> Clone for _Inner<T> {
- fn clone(&self) -> Self {
- Self(self.0.clone(), self.1.clone())
- }
- }
- impl<T: std::fmt::Debug> std::fmt::Debug for _Inner<T> {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- write!(f, "{:?}", self.0)
- }
- }
- impl<T: FlightService> tonic::transport::NamedService for FlightServiceServer<T> {
- const NAME: &'static str = "arrow.flight.protocol.FlightService";
- }
-}
diff --git a/rust/arrow-flight/src/lib.rs b/rust/arrow-flight/src/lib.rs
deleted file mode 100644
index 6af2e74..0000000
--- a/rust/arrow-flight/src/lib.rs
+++ /dev/null
@@ -1,20 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-include!("arrow.flight.protocol.rs");
-
-pub mod utils;
diff --git a/rust/arrow-flight/src/utils.rs b/rust/arrow-flight/src/utils.rs
deleted file mode 100644
index 659668c..0000000
--- a/rust/arrow-flight/src/utils.rs
+++ /dev/null
@@ -1,167 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Utilities to assist with reading and writing Arrow data as Flight messages
-
-use std::convert::TryFrom;
-
-use crate::{FlightData, SchemaResult};
-
-use arrow::array::ArrayRef;
-use arrow::datatypes::{Schema, SchemaRef};
-use arrow::error::{ArrowError, Result};
-use arrow::ipc::{convert, reader, writer, writer::EncodedData, writer::IpcWriteOptions};
-use arrow::record_batch::RecordBatch;
-
-/// Convert a `RecordBatch` to a vector of `FlightData` representing the bytes of the dictionaries
-/// and a `FlightData` representing the bytes of the batch's values
-pub fn flight_data_from_arrow_batch(
- batch: &RecordBatch,
- options: &IpcWriteOptions,
-) -> (Vec<FlightData>, FlightData) {
- let data_gen = writer::IpcDataGenerator::default();
- let mut dictionary_tracker = writer::DictionaryTracker::new(false);
-
- let (encoded_dictionaries, encoded_batch) = data_gen
- .encoded_batch(batch, &mut dictionary_tracker, &options)
- .expect("DictionaryTracker configured above to not error on replacement");
-
- let flight_dictionaries = encoded_dictionaries.into_iter().map(Into::into).collect();
- let flight_batch = encoded_batch.into();
-
- (flight_dictionaries, flight_batch)
-}
-
-impl From<EncodedData> for FlightData {
- fn from(data: EncodedData) -> Self {
- FlightData {
- data_header: data.ipc_message,
- data_body: data.arrow_data,
- ..Default::default()
- }
- }
-}
-
-/// Convert a `Schema` to `SchemaResult` by converting to an IPC message
-pub fn flight_schema_from_arrow_schema(
- schema: &Schema,
- options: &IpcWriteOptions,
-) -> SchemaResult {
- SchemaResult {
- schema: flight_schema_as_flatbuffer(schema, options),
- }
-}
-
-/// Convert a `Schema` to `FlightData` by converting to an IPC message
-pub fn flight_data_from_arrow_schema(
- schema: &Schema,
- options: &IpcWriteOptions,
-) -> FlightData {
- let data_header = flight_schema_as_flatbuffer(schema, options);
- FlightData {
- data_header,
- ..Default::default()
- }
-}
-
-/// Convert a `Schema` to bytes in the format expected in `FlightInfo.schema`
-pub fn ipc_message_from_arrow_schema(
- arrow_schema: &Schema,
- options: &IpcWriteOptions,
-) -> Result<Vec<u8>> {
- let encoded_data = flight_schema_as_encoded_data(arrow_schema, options);
-
- let mut schema = vec![];
- arrow::ipc::writer::write_message(&mut schema, encoded_data, options)?;
- Ok(schema)
-}
-
-fn flight_schema_as_flatbuffer(
- arrow_schema: &Schema,
- options: &IpcWriteOptions,
-) -> Vec<u8> {
- let encoded_data = flight_schema_as_encoded_data(arrow_schema, options);
- encoded_data.ipc_message
-}
-
-fn flight_schema_as_encoded_data(
- arrow_schema: &Schema,
- options: &IpcWriteOptions,
-) -> EncodedData {
- let data_gen = writer::IpcDataGenerator::default();
- data_gen.schema_to_bytes(arrow_schema, options)
-}
-
-/// Try convert `FlightData` into an Arrow Schema
-///
-/// Returns an error if the `FlightData` header is not a valid IPC schema
-impl TryFrom<&FlightData> for Schema {
- type Error = ArrowError;
- fn try_from(data: &FlightData) -> Result<Self> {
- convert::schema_from_bytes(&data.data_header[..]).map_err(|err| {
- ArrowError::ParseError(format!(
- "Unable to convert flight data to Arrow schema: {}",
- err
- ))
- })
- }
-}
-
-/// Try convert `SchemaResult` into an Arrow Schema
-///
-/// Returns an error if the `FlightData` header is not a valid IPC schema
-impl TryFrom<&SchemaResult> for Schema {
- type Error = ArrowError;
- fn try_from(data: &SchemaResult) -> Result<Self> {
- convert::schema_from_bytes(&data.schema[..]).map_err(|err| {
- ArrowError::ParseError(format!(
- "Unable to convert schema result to Arrow schema: {}",
- err
- ))
- })
- }
-}
-
-/// Convert a FlightData message to a RecordBatch
-pub fn flight_data_to_arrow_batch(
- data: &FlightData,
- schema: SchemaRef,
- dictionaries_by_field: &[Option<ArrayRef>],
-) -> Result<RecordBatch> {
- // check that the data_header is a record batch message
- let message = arrow::ipc::root_as_message(&data.data_header[..]).map_err(|err| {
- ArrowError::ParseError(format!("Unable to get root as message: {:?}", err))
- })?;
-
- message
- .header_as_record_batch()
- .ok_or_else(|| {
- ArrowError::ParseError(
- "Unable to convert flight data header to a record batch".to_string(),
- )
- })
- .map(|batch| {
- reader::read_record_batch(
- &data.data_body,
- batch,
- schema,
- &dictionaries_by_field,
- )
- })?
-}
-
-// TODO: add more explicit conversion that exposes flight descriptor and metadata options
diff --git a/rust/arrow-pyarrow-integration-testing/.cargo/config b/rust/arrow-pyarrow-integration-testing/.cargo/config
deleted file mode 100644
index a127967..0000000
--- a/rust/arrow-pyarrow-integration-testing/.cargo/config
+++ /dev/null
@@ -1,22 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-[target.x86_64-apple-darwin]
-rustflags = [
- "-C", "link-arg=-undefined",
- "-C", "link-arg=dynamic_lookup",
-]
\ No newline at end of file
diff --git a/rust/arrow-pyarrow-integration-testing/.gitignore b/rust/arrow-pyarrow-integration-testing/.gitignore
deleted file mode 100644
index 82adb58..0000000
--- a/rust/arrow-pyarrow-integration-testing/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-__pycache__
-venv
diff --git a/rust/arrow-pyarrow-integration-testing/Cargo.toml b/rust/arrow-pyarrow-integration-testing/Cargo.toml
deleted file mode 100644
index ef356b0..0000000
--- a/rust/arrow-pyarrow-integration-testing/Cargo.toml
+++ /dev/null
@@ -1,38 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-[package]
-name = "arrow-pyarrow-integration-testing"
-description = ""
-version = "4.0.0-SNAPSHOT"
-homepage = "https://github.com/apache/arrow"
-repository = "https://github.com/apache/arrow"
-authors = ["Apache Arrow <de...@arrow.apache.org>"]
-license = "Apache-2.0"
-keywords = [ "arrow" ]
-edition = "2018"
-
-[lib]
-name = "arrow_pyarrow_integration_testing"
-crate-type = ["cdylib"]
-
-[dependencies]
-arrow = { path = "../arrow", version = "4.0.0-SNAPSHOT" }
-pyo3 = { version = "0.12.1", features = ["extension-module"] }
-
-[package.metadata.maturin]
-requires-dist = ["pyarrow>=1"]
diff --git a/rust/arrow-pyarrow-integration-testing/README.md b/rust/arrow-pyarrow-integration-testing/README.md
deleted file mode 100644
index 7e78aa9..0000000
--- a/rust/arrow-pyarrow-integration-testing/README.md
+++ /dev/null
@@ -1,57 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-# Arrow c integration
-
-This is a Rust crate that tests compatibility between Rust's Arrow implementation and PyArrow.
-
-Note that this crate uses two languages and an external ABI:
-* `Rust`
-* `Python`
-* C ABI privately exposed by `Pyarrow`.
-
-## Basic idea
-
-Pyarrow exposes a C ABI to convert arrow arrays from and to its C implementation, see [here](https://arrow.apache.org/docs/format/CDataInterface.html).
-
-This package uses the equivalent struct in Rust (`arrow::array::ArrowArray`), and verifies that
-we can use pyarrow's interface to move pointers from and to Rust.
-
-## Relevant literature
-
-* [Arrow's CDataInterface](https://arrow.apache.org/docs/format/CDataInterface.html)
-* [Rust's FFI](https://doc.rust-lang.org/nomicon/ffi.html)
-* [Pyarrow private binds](https://github.com/apache/arrow/blob/ae1d24efcc3f1ac2a876d8d9f544a34eb04ae874/python/pyarrow/array.pxi#L1226)
-* [PyO3](https://docs.rs/pyo3/0.12.1/pyo3/index.html)
-
-## How to develop
-
-```bash
-# prepare development environment (used to build wheel / install in development)
-python -m venv venv
-venv/bin/pip install maturin==0.8.2 toml==0.10.1 pyarrow==1.0.0
-```
-
-Whenever rust code changes (your changes or via git pull):
-
-```bash
-source venv/bin/activate
-maturin develop
-python -m unittest discover tests
-```
diff --git a/rust/arrow-pyarrow-integration-testing/pyproject.toml b/rust/arrow-pyarrow-integration-testing/pyproject.toml
deleted file mode 100644
index 2748069..0000000
--- a/rust/arrow-pyarrow-integration-testing/pyproject.toml
+++ /dev/null
@@ -1,20 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-[build-system]
-requires = ["maturin"]
-build-backend = "maturin"
diff --git a/rust/arrow-pyarrow-integration-testing/src/lib.rs b/rust/arrow-pyarrow-integration-testing/src/lib.rs
deleted file mode 100644
index 5b5462d..0000000
--- a/rust/arrow-pyarrow-integration-testing/src/lib.rs
+++ /dev/null
@@ -1,188 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! This library demonstrates a minimal usage of Rust's C data interface to pass
-//! arrays from and to Python.
-
-use std::error;
-use std::fmt;
-use std::sync::Arc;
-
-use pyo3::exceptions::PyOSError;
-use pyo3::wrap_pyfunction;
-use pyo3::{libc::uintptr_t, prelude::*};
-
-use arrow::array::{make_array_from_raw, ArrayRef, Int64Array};
-use arrow::compute::kernels;
-use arrow::error::ArrowError;
-use arrow::ffi;
-
-/// an error that bridges ArrowError with a Python error
-#[derive(Debug)]
-enum PyO3ArrowError {
- ArrowError(ArrowError),
-}
-
-impl fmt::Display for PyO3ArrowError {
- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- match *self {
- PyO3ArrowError::ArrowError(ref e) => e.fmt(f),
- }
- }
-}
-
-impl error::Error for PyO3ArrowError {
- fn source(&self) -> Option<&(dyn error::Error + 'static)> {
- match *self {
- // The cause is the underlying implementation error type. Is implicitly
- // cast to the trait object `&error::Error`. This works because the
- // underlying type already implements the `Error` trait.
- PyO3ArrowError::ArrowError(ref e) => Some(e),
- }
- }
-}
-
-impl From<ArrowError> for PyO3ArrowError {
- fn from(err: ArrowError) -> PyO3ArrowError {
- PyO3ArrowError::ArrowError(err)
- }
-}
-
-impl From<PyO3ArrowError> for PyErr {
- fn from(err: PyO3ArrowError) -> PyErr {
- PyOSError::new_err(err.to_string())
- }
-}
-
-fn to_rust(ob: PyObject, py: Python) -> PyResult<ArrayRef> {
- // prepare a pointer to receive the Array struct
- let (array_pointer, schema_pointer) =
- ffi::ArrowArray::into_raw(unsafe { ffi::ArrowArray::empty() });
-
- // make the conversion through PyArrow's private API
- // this changes the pointer's memory and is thus unsafe. In particular, `_export_to_c` can go out of bounds
- ob.call_method1(
- py,
- "_export_to_c",
- (array_pointer as uintptr_t, schema_pointer as uintptr_t),
- )?;
-
- let array = unsafe { make_array_from_raw(array_pointer, schema_pointer) }
- .map_err(|e| PyO3ArrowError::from(e))?;
- Ok(array)
-}
-
-fn to_py(array: ArrayRef, py: Python) -> PyResult<PyObject> {
- let (array_pointer, schema_pointer) =
- array.to_raw().map_err(|e| PyO3ArrowError::from(e))?;
-
- let pa = py.import("pyarrow")?;
-
- let array = pa.getattr("Array")?.call_method1(
- "_import_from_c",
- (array_pointer as uintptr_t, schema_pointer as uintptr_t),
- )?;
- Ok(array.to_object(py))
-}
-
-/// Returns `array + array` of an int64 array.
-#[pyfunction]
-fn double(array: PyObject, py: Python) -> PyResult<PyObject> {
- // import
- let array = to_rust(array, py)?;
-
- // perform some operation
- let array =
- array
- .as_any()
- .downcast_ref::<Int64Array>()
- .ok_or(PyO3ArrowError::ArrowError(ArrowError::ParseError(
- "Expects an int64".to_string(),
- )))?;
- let array =
- kernels::arithmetic::add(&array, &array).map_err(|e| PyO3ArrowError::from(e))?;
- let array = Arc::new(array);
-
- // export
- to_py(array, py)
-}
-
-/// calls a lambda function that receives and returns an array
-/// whose result must be the array multiplied by two
-#[pyfunction]
-fn double_py(lambda: PyObject, py: Python) -> PyResult<bool> {
- // create
- let array = Arc::new(Int64Array::from(vec![Some(1), None, Some(3)]));
- let expected = Arc::new(Int64Array::from(vec![Some(2), None, Some(6)])) as ArrayRef;
-
- // to py
- let array = to_py(array, py)?;
-
- let array = lambda.call1(py, (array,))?;
-
- let array = to_rust(array, py)?;
-
- Ok(array == expected)
-}
-
-/// Returns the substring
-#[pyfunction]
-fn substring(array: PyObject, start: i64, py: Python) -> PyResult<PyObject> {
- // import
- let array = to_rust(array, py)?;
-
- // substring
- let array = kernels::substring::substring(array.as_ref(), start, &None)
- .map_err(|e| PyO3ArrowError::from(e))?;
-
- // export
- to_py(array, py)
-}
-
-/// Returns the concatenate
-#[pyfunction]
-fn concatenate(array: PyObject, py: Python) -> PyResult<PyObject> {
- // import
- let array = to_rust(array, py)?;
-
- // concat
- let array = kernels::concat::concat(&[array.as_ref(), array.as_ref()])
- .map_err(|e| PyO3ArrowError::from(e))?;
-
- // export
- to_py(array, py)
-}
-
-/// Converts to rust and back to python
-#[pyfunction]
-fn round_trip(array: PyObject, py: Python) -> PyResult<PyObject> {
- // import
- let array = to_rust(array, py)?;
-
- // export
- to_py(array, py)
-}
-
-#[pymodule]
-fn arrow_pyarrow_integration_testing(_py: Python, m: &PyModule) -> PyResult<()> {
- m.add_wrapped(wrap_pyfunction!(double))?;
- m.add_wrapped(wrap_pyfunction!(double_py))?;
- m.add_wrapped(wrap_pyfunction!(substring))?;
- m.add_wrapped(wrap_pyfunction!(concatenate))?;
- m.add_wrapped(wrap_pyfunction!(round_trip))?;
- Ok(())
-}
diff --git a/rust/arrow-pyarrow-integration-testing/tests/test_sql.py b/rust/arrow-pyarrow-integration-testing/tests/test_sql.py
deleted file mode 100644
index c0de382..0000000
--- a/rust/arrow-pyarrow-integration-testing/tests/test_sql.py
+++ /dev/null
@@ -1,99 +0,0 @@
-# -*- coding: utf-8 -*-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import unittest
-
-import pyarrow
-import arrow_pyarrow_integration_testing
-
-
-class TestCase(unittest.TestCase):
- def test_primitive_python(self):
- """
- Python -> Rust -> Python
- """
- old_allocated = pyarrow.total_allocated_bytes()
- a = pyarrow.array([1, 2, 3])
- b = arrow_pyarrow_integration_testing.double(a)
- self.assertEqual(b, pyarrow.array([2, 4, 6]))
- del a
- del b
- # No leak of C++ memory
- self.assertEqual(old_allocated, pyarrow.total_allocated_bytes())
-
- def test_primitive_rust(self):
- """
- Rust -> Python -> Rust
- """
- old_allocated = pyarrow.total_allocated_bytes()
-
- def double(array):
- array = array.to_pylist()
- return pyarrow.array([x * 2 if x is not None else None for x in array])
-
- is_correct = arrow_pyarrow_integration_testing.double_py(double)
- self.assertTrue(is_correct)
- # No leak of C++ memory
- self.assertEqual(old_allocated, pyarrow.total_allocated_bytes())
-
- def test_string_python(self):
- """
- Python -> Rust -> Python
- """
- old_allocated = pyarrow.total_allocated_bytes()
- a = pyarrow.array(["a", None, "ccc"])
- b = arrow_pyarrow_integration_testing.substring(a, 1)
- self.assertEqual(b, pyarrow.array(["", None, "cc"]))
- del a
- del b
- # No leak of C++ memory
- self.assertEqual(old_allocated, pyarrow.total_allocated_bytes())
-
- def test_time32_python(self):
- """
- Python -> Rust -> Python
- """
- old_allocated = pyarrow.total_allocated_bytes()
- a = pyarrow.array([None, 1, 2], pyarrow.time32('s'))
- b = arrow_pyarrow_integration_testing.concatenate(a)
- expected = pyarrow.array([None, 1, 2] + [None, 1, 2], pyarrow.time32('s'))
- self.assertEqual(b, expected)
- del a
- del b
- del expected
- # No leak of C++ memory
- self.assertEqual(old_allocated, pyarrow.total_allocated_bytes())
-
- def test_list_array(self):
- """
- Python -> Rust -> Python
- """
- old_allocated = pyarrow.total_allocated_bytes()
- a = pyarrow.array([[], None, [1, 2], [4, 5, 6]], pyarrow.list_(pyarrow.int64()))
- b = arrow_pyarrow_integration_testing.round_trip(a)
-
- b.validate(full=True)
- assert a.to_pylist() == b.to_pylist()
- assert a.type == b.type
- del a
- del b
- # No leak of C++ memory
- self.assertEqual(old_allocated, pyarrow.total_allocated_bytes())
-
-
-
diff --git a/rust/arrow/Cargo.toml b/rust/arrow/Cargo.toml
deleted file mode 100644
index 5ab1f8c..0000000
--- a/rust/arrow/Cargo.toml
+++ /dev/null
@@ -1,151 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-[package]
-name = "arrow"
-version = "4.0.0-SNAPSHOT"
-description = "Rust implementation of Apache Arrow"
-homepage = "https://github.com/apache/arrow"
-repository = "https://github.com/apache/arrow"
-authors = ["Apache Arrow <de...@arrow.apache.org>"]
-license = "Apache-2.0"
-keywords = [ "arrow" ]
-include = [
- "benches/*.rs",
- "src/**/*.rs",
- "Cargo.toml",
-]
-edition = "2018"
-
-[lib]
-name = "arrow"
-path = "src/lib.rs"
-
-[dependencies]
-serde = { version = "1.0", features = ["rc"] }
-serde_derive = "1.0"
-serde_json = { version = "1.0", features = ["preserve_order"] }
-indexmap = "1.6"
-rand = "0.7"
-csv = "1.1"
-num = "0.3"
-regex = "1.3"
-lazy_static = "1.4"
-packed_simd = { version = "0.3.4", optional = true, package = "packed_simd_2" }
-chrono = "0.4"
-flatbuffers = "^0.8"
-hex = "0.4"
-prettytable-rs = { version = "0.8.0", optional = true }
-lexical-core = "^0.7"
-
-[features]
-default = []
-avx512 = []
-simd = ["packed_simd"]
-prettyprint = ["prettytable-rs"]
-# this is only intended to be used in single-threaded programs: it verifies that
-# all allocated memory is being released (no memory leaks).
-# See README for details
-memory-check = []
-
-[dev-dependencies]
-criterion = "0.3"
-flate2 = "1"
-tempfile = "3"
-
-[build-dependencies]
-cfg_aliases = "0.1"
-
-[[bench]]
-name = "aggregate_kernels"
-harness = false
-
-[[bench]]
-name = "array_from_vec"
-harness = false
-
-[[bench]]
-name = "builder"
-harness = false
-
-[[bench]]
-name = "buffer_bit_ops"
-harness = false
-
-[[bench]]
-name = "boolean_kernels"
-harness = false
-
-[[bench]]
-name = "arithmetic_kernels"
-harness = false
-
-[[bench]]
-name = "cast_kernels"
-harness = false
-
-[[bench]]
-name = "comparison_kernels"
-harness = false
-
-[[bench]]
-name = "filter_kernels"
-harness = false
-
-[[bench]]
-name = "take_kernels"
-harness = false
-
-[[bench]]
-name = "length_kernel"
-harness = false
-
-[[bench]]
-name = "bit_length_kernel"
-harness = false
-
-[[bench]]
-name = "sort_kernel"
-harness = false
-
-[[bench]]
-name = "csv_writer"
-harness = false
-
-[[bench]]
-name = "json_reader"
-harness = false
-
-[[bench]]
-name = "equal"
-harness = false
-
-[[bench]]
-name = "array_slice"
-harness = false
-
-[[bench]]
-name = "concatenate_kernel"
-harness = false
-
-[[bench]]
-name = "mutable_array"
-harness = false
-
-[[bench]]
-name = "buffer_create"
-harness = false
diff --git a/rust/arrow/README.md b/rust/arrow/README.md
deleted file mode 100644
index 674c3fc..0000000
--- a/rust/arrow/README.md
+++ /dev/null
@@ -1,206 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-# Native Rust implementation of Apache Arrow
-
-This crate contains a native Rust implementation of the [Arrow columnar format](https://arrow.apache.org/docs/format/Columnar.html).
-
-## Developer's guide
-
-Common information for all Rust libraries in this project, including
-testing, code formatting, and lints, can be found in the main Arrow
-Rust [README.md](../README.md).
-
-Please refer to [lib.rs](src/lib.rs) for an introduction to this
-specific crate and its current functionality.
-
-### How to check memory allocations
-
-This crate heavily uses `unsafe` due to how memory is allocated in cache lines.
-We have a small tool to verify that this crate does not leak memory (beyond what the compiler already does)
-
-Run it with
-
-```bash
-cargo test --features memory-check --lib -- --test-threads 1
-```
-
-This runs all unit-tests on a single thread and counts all allocations and de-allocations.
-
-## Examples
-
-The examples folder shows how to construct some different types of Arrow
-arrays, including dynamic arrays created at runtime.
-
-Examples can be run using the `cargo run --example` command. For example:
-
-```bash
-cargo run --example builders
-cargo run --example dynamic_types
-cargo run --example read_csv
-```
-
-## IPC
-
-The expected flatc version is 1.12.0+, built from [flatbuffers](https://github.com/google/flatbuffers)
-master at fixed commit ID, by regen.sh.
-
-The IPC flatbuffer code was generated by running this command from the root of the project:
-
-```bash
-./regen.sh
-```
-
-The above script will run the `flatc` compiler and perform some adjustments to the source code:
-
-- Replace `type__` with `type_`
-- Remove `org::apache::arrow::flatbuffers` namespace
-- Add includes to each generated file
-
-## Features
-
-Arrow uses the following features:
-
-* `simd` - Arrow uses the [packed_simd](https://crates.io/crates/packed_simd) crate to optimize many of the
- implementations in the [compute](https://github.com/apache/arrow/tree/master/rust/arrow/src/compute)
- module using SIMD intrinsics. These optimizations are turned *off* by default.
- If the `simd` feature is enabled, an unstable version of Rust is required (we test with `nightly-2021-03-24`)
-* `flight` which contains useful functions to convert between the Flight wire format and Arrow data
-* `prettyprint` which is a utility for printing record batches
-
-Other than `simd` all the other features are enabled by default. Disabling `prettyprint` might be necessary in order to
-compile Arrow to the `wasm32-unknown-unknown` WASM target.
-
-## Guidelines in usage of `unsafe`
-
-[`unsafe`](https://doc.rust-lang.org/book/ch19-01-unsafe-rust.html) has a high maintenance cost because debugging and testing it is difficult, time consuming, often requires external tools (e.g. `valgrind`), and requires a higher-than-usual attention to details. Undefined behavior is particularly difficult to identify and test, and usage of `unsafe` is the [primary cause of undefined behavior](https://doc.rust-lang.org/reference/behavior-considered-undefined.html) in a program written in Rust.
-For two real world examples of where `unsafe` has consumed time in the past in this project see [#8545](https://github.com/apache/arrow/pull/8645) and [8829](https://github.com/apache/arrow/pull/8829)
-This crate only accepts the usage of `unsafe` code upon careful consideration, and strives to avoid it to the largest possible extent.
-
-### When can `unsafe` be used?
-
-Generally, `unsafe` should only be used when a `safe` counterpart is not available and there is no `safe` way to achieve additional performance in that area. The following is a summary of the current components of the crate that require `unsafe`:
-
-* alloc, dealloc and realloc of buffers along cache lines
-* Interpreting bytes as certain rust types, for access, representation and compute
-* Foreign interfaces (C data interface)
-* Inter-process communication (IPC)
-* SIMD
-* Performance (e.g. omit bounds checks, use of pointers to avoid bound checks)
-
-#### cache-line aligned memory management
-
-The arrow format recommends storing buffers aligned with cache lines, and this crate adopts this behavior.
-However, Rust's global allocator does not allocate memory aligned with cache-lines. As such, many of the low-level operations related to memory management require `unsafe`.
-
-#### Interpreting bytes
-
-The arrow format is specified in bytes (`u8`), which can be logically represented as certain types
-depending on the `DataType`.
-For many operations, such as access, representation, numerical computation and string manipulation,
-it is often necessary to interpret bytes as other physical types (e.g. `i32`).
-
-Usage of `unsafe` for the purpose of interpreting bytes in their corresponding type (according to the arrow specification) is allowed. Specifically, the pointer to the byte slice must be aligned to the type that it intends to represent and the length of the slice is a multiple of the size of the target type of the transmutation.
-
-#### FFI
-
-The arrow format declares an ABI for zero-copy from and to libraries that implement the specification
-(foreign interfaces). In Rust, receiving and sending pointers via FFI requires usage of `unsafe` due to
-the impossibility of the compiler to derive the invariants (such as lifetime, null pointers, and pointer alignment) from the source code alone as they are part of the FFI contract.
-
-#### IPC
-
-The arrow format declares a IPC protocol, which this crate supports. IPC is equivalent to a FFI in that the rust compiler can't reason about the contract's invariants.
-
-#### SIMD
-
-The API provided by the `packed_simd` library is currently `unsafe`. However, SIMD offers a significant performance improvement over non-SIMD operations.
-
-#### Performance
-
-Some operations are significantly faster when `unsafe` is used.
-
-A common usage of `unsafe` is to offer an API to access the `i`th element of an array (e.g. `UInt32Array`).
-This requires accessing the values buffer e.g. `array.buffers()[0]`, picking the slice
-`[i * size_of<i32>(), (i + 1) * size_of<i32>()]`, and then transmuting it to `i32`. In safe Rust,
-this operation requires boundary checks that are detrimental to performance.
-
-Usage of `unsafe` for performance reasons is justified only when all other alternatives have been exhausted and the performance benefits are sufficiently large (e.g. >~10%).
-
-### Considerations when introducing `unsafe`
-
-Usage of `unsafe` in this crate *must*:
-
-* not expose a public API as `safe` when there are necessary invariants for that API to be defined behavior.
-* have code documentation for why `safe` is not used / possible
-* have code documentation about which invariant the user needs to enforce to ensure [soundness](https://rust-lang.github.io/unsafe-code-guidelines/glossary.html#soundness-of-code--of-a-library), or which
-* invariant is being preserved.
-* if applicable, use `debug_assert`s to relevant invariants (e.g. bound checks)
-
-Example of code documentation:
-
-```rust
-// JUSTIFICATION
-// Benefit
-// Describe the benefit of using unsafe. E.g.
-// "30% performance degradation if the safe counterpart is used, see bench X."
-// Soundness
-// Describe why the code remains sound (according to the definition of rust's unsafe code guidelines). E.g.
-// "We bounded check these values at initialization and the array is immutable."
-let ... = unsafe { ... };
-```
-
-When adding this documentation to existing code that is not sound and cannot trivially be fixed, we should file
-specific JIRA issues and reference them in these code comments. For example:
-
-```rust
-// Soundness
-// This is not sound because .... see https://issues.apache.org/jira/browse/ARROW-nnnnn
-```
-
-# Publishing to crates.io
-
-An Arrow committer can publish this crate after an official project release has
-been made to crates.io using the following instructions.
-
-Follow [these
-instructions](https://doc.rust-lang.org/cargo/reference/publishing.html) to
-create an account and login to crates.io before asking to be added as an owner
-of the [arrow crate](https://crates.io/crates/arrow).
-
-Checkout the tag for the version to be released. For example:
-
-```bash
-git checkout apache-arrow-0.11.0
-```
-
-If the Cargo.toml in this tag already contains `version = "0.11.0"` (as it
-should) then the crate can be published with the following command:
-
-```bash
-cargo publish
-```
-
-If the Cargo.toml does not have the correct version then it will be necessary
-to modify it manually. Since there is now a modified file locally that is not
-committed to GitHub it will be necessary to use the following command.
-
-```bash
-cargo publish --allow-dirty
-```
diff --git a/rust/arrow/benches/aggregate_kernels.rs b/rust/arrow/benches/aggregate_kernels.rs
deleted file mode 100644
index 1724b73..0000000
--- a/rust/arrow/benches/aggregate_kernels.rs
+++ /dev/null
@@ -1,67 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#[macro_use]
-extern crate criterion;
-use criterion::Criterion;
-
-extern crate arrow;
-
-use arrow::compute::kernels::aggregate::*;
-use arrow::util::bench_util::*;
-use arrow::{array::*, datatypes::Float32Type};
-
-fn bench_sum(arr_a: &Float32Array) {
- criterion::black_box(sum(&arr_a).unwrap());
-}
-
-fn bench_min(arr_a: &Float32Array) {
- criterion::black_box(min(&arr_a).unwrap());
-}
-
-fn bench_max(arr_a: &Float32Array) {
- criterion::black_box(max(&arr_a).unwrap());
-}
-
-fn bench_min_string(arr_a: &StringArray) {
- criterion::black_box(min_string(&arr_a).unwrap());
-}
-
-fn add_benchmark(c: &mut Criterion) {
- let arr_a = create_primitive_array::<Float32Type>(512, 0.0);
-
- c.bench_function("sum 512", |b| b.iter(|| bench_sum(&arr_a)));
- c.bench_function("min 512", |b| b.iter(|| bench_min(&arr_a)));
- c.bench_function("max 512", |b| b.iter(|| bench_max(&arr_a)));
-
- let arr_a = create_primitive_array::<Float32Type>(512, 0.5);
-
- c.bench_function("sum nulls 512", |b| b.iter(|| bench_sum(&arr_a)));
- c.bench_function("min nulls 512", |b| b.iter(|| bench_min(&arr_a)));
- c.bench_function("max nulls 512", |b| b.iter(|| bench_max(&arr_a)));
-
- let arr_b = create_string_array::<i32>(512, 0.0);
- c.bench_function("min string 512", |b| b.iter(|| bench_min_string(&arr_b)));
-
- let arr_b = create_string_array::<i32>(512, 0.5);
- c.bench_function("min nulls string 512", |b| {
- b.iter(|| bench_min_string(&arr_b))
- });
-}
-
-criterion_group!(benches, add_benchmark);
-criterion_main!(benches);
diff --git a/rust/arrow/benches/arithmetic_kernels.rs b/rust/arrow/benches/arithmetic_kernels.rs
deleted file mode 100644
index 721157e..0000000
--- a/rust/arrow/benches/arithmetic_kernels.rs
+++ /dev/null
@@ -1,103 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#[macro_use]
-extern crate criterion;
-use criterion::Criterion;
-use rand::Rng;
-
-use std::sync::Arc;
-
-extern crate arrow;
-
-use arrow::compute::kernels::limit::*;
-use arrow::util::bench_util::*;
-use arrow::{array::*, datatypes::Float32Type};
-use arrow::{compute::kernels::arithmetic::*, util::test_util::seedable_rng};
-
-fn create_array(size: usize, with_nulls: bool) -> ArrayRef {
- let null_density = if with_nulls { 0.5 } else { 0.0 };
- let array = create_primitive_array::<Float32Type>(size, null_density);
- Arc::new(array)
-}
-
-fn bench_add(arr_a: &ArrayRef, arr_b: &ArrayRef) {
- let arr_a = arr_a.as_any().downcast_ref::<Float32Array>().unwrap();
- let arr_b = arr_b.as_any().downcast_ref::<Float32Array>().unwrap();
- criterion::black_box(add(arr_a, arr_b).unwrap());
-}
-
-fn bench_subtract(arr_a: &ArrayRef, arr_b: &ArrayRef) {
- let arr_a = arr_a.as_any().downcast_ref::<Float32Array>().unwrap();
- let arr_b = arr_b.as_any().downcast_ref::<Float32Array>().unwrap();
- criterion::black_box(subtract(&arr_a, &arr_b).unwrap());
-}
-
-fn bench_multiply(arr_a: &ArrayRef, arr_b: &ArrayRef) {
- let arr_a = arr_a.as_any().downcast_ref::<Float32Array>().unwrap();
- let arr_b = arr_b.as_any().downcast_ref::<Float32Array>().unwrap();
- criterion::black_box(multiply(&arr_a, &arr_b).unwrap());
-}
-
-fn bench_divide(arr_a: &ArrayRef, arr_b: &ArrayRef) {
- let arr_a = arr_a.as_any().downcast_ref::<Float32Array>().unwrap();
- let arr_b = arr_b.as_any().downcast_ref::<Float32Array>().unwrap();
- criterion::black_box(divide(&arr_a, &arr_b).unwrap());
-}
-
-fn bench_divide_scalar(array: &ArrayRef, divisor: f32) {
- let array = array.as_any().downcast_ref::<Float32Array>().unwrap();
- criterion::black_box(divide_scalar(&array, divisor).unwrap());
-}
-
-fn bench_limit(arr_a: &ArrayRef, max: usize) {
- criterion::black_box(limit(arr_a, max));
-}
-
-fn add_benchmark(c: &mut Criterion) {
- let arr_a = create_array(512, false);
- let arr_b = create_array(512, false);
- let scalar = seedable_rng().gen();
-
- c.bench_function("add 512", |b| b.iter(|| bench_add(&arr_a, &arr_b)));
- c.bench_function("subtract 512", |b| {
- b.iter(|| bench_subtract(&arr_a, &arr_b))
- });
- c.bench_function("multiply 512", |b| {
- b.iter(|| bench_multiply(&arr_a, &arr_b))
- });
- c.bench_function("divide 512", |b| b.iter(|| bench_divide(&arr_a, &arr_b)));
- c.bench_function("divide_scalar 512", |b| {
- b.iter(|| bench_divide_scalar(&arr_a, scalar))
- });
- c.bench_function("limit 512, 512", |b| b.iter(|| bench_limit(&arr_a, 512)));
-
- let arr_a_nulls = create_array(512, false);
- let arr_b_nulls = create_array(512, false);
- c.bench_function("add_nulls_512", |b| {
- b.iter(|| bench_add(&arr_a_nulls, &arr_b_nulls))
- });
- c.bench_function("divide_nulls_512", |b| {
- b.iter(|| bench_divide(&arr_a_nulls, &arr_b_nulls))
- });
- c.bench_function("divide_scalar_nulls_512", |b| {
- b.iter(|| bench_divide_scalar(&arr_a_nulls, scalar))
- });
-}
-
-criterion_group!(benches, add_benchmark);
-criterion_main!(benches);
diff --git a/rust/arrow/benches/array_from_vec.rs b/rust/arrow/benches/array_from_vec.rs
deleted file mode 100644
index 7740c6b..0000000
--- a/rust/arrow/benches/array_from_vec.rs
+++ /dev/null
@@ -1,120 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#[macro_use]
-extern crate criterion;
-use criterion::Criterion;
-
-extern crate arrow;
-
-use arrow::array::*;
-use arrow::buffer::Buffer;
-use arrow::datatypes::*;
-use std::{convert::TryFrom, sync::Arc};
-
-fn array_from_vec(n: usize) {
- let mut v: Vec<u8> = Vec::with_capacity(n);
- for i in 0..n {
- v.push((i & 0xffff) as u8);
- }
- let arr_data = ArrayDataBuilder::new(DataType::Int32)
- .add_buffer(Buffer::from(v))
- .build();
- criterion::black_box(Int32Array::from(arr_data));
-}
-
-fn array_string_from_vec(n: usize) {
- let mut v: Vec<Option<&str>> = Vec::with_capacity(n);
- for i in 0..n {
- if i % 2 == 0 {
- v.push(Some("hello world"));
- } else {
- v.push(None);
- }
- }
- criterion::black_box(StringArray::from(v));
-}
-
-fn struct_array_values(
- n: usize,
-) -> (
- &'static str,
- Vec<Option<&'static str>>,
- &'static str,
- Vec<Option<i32>>,
-) {
- let mut strings: Vec<Option<&str>> = Vec::with_capacity(n);
- let mut ints: Vec<Option<i32>> = Vec::with_capacity(n);
- for _ in 0..n / 4 {
- strings.extend_from_slice(&[Some("joe"), None, None, Some("mark")]);
- ints.extend_from_slice(&[Some(1), Some(2), None, Some(4)]);
- }
- ("f1", strings, "f2", ints)
-}
-
-fn struct_array_from_vec(
- field1: &str,
- strings: &[Option<&str>],
- field2: &str,
- ints: &[Option<i32>],
-) {
- let strings: ArrayRef = Arc::new(StringArray::from(strings.to_owned()));
- let ints: ArrayRef = Arc::new(Int32Array::from(ints.to_owned()));
-
- criterion::black_box(
- StructArray::try_from(vec![(field1, strings), (field2, ints)]).unwrap(),
- );
-}
-
-fn criterion_benchmark(c: &mut Criterion) {
- c.bench_function("array_from_vec 128", |b| b.iter(|| array_from_vec(128)));
- c.bench_function("array_from_vec 256", |b| b.iter(|| array_from_vec(256)));
- c.bench_function("array_from_vec 512", |b| b.iter(|| array_from_vec(512)));
-
- c.bench_function("array_string_from_vec 128", |b| {
- b.iter(|| array_string_from_vec(128))
- });
- c.bench_function("array_string_from_vec 256", |b| {
- b.iter(|| array_string_from_vec(256))
- });
- c.bench_function("array_string_from_vec 512", |b| {
- b.iter(|| array_string_from_vec(512))
- });
-
- let (field1, strings, field2, ints) = struct_array_values(128);
- c.bench_function("struct_array_from_vec 128", |b| {
- b.iter(|| struct_array_from_vec(&field1, &strings, &field2, &ints))
- });
-
- let (field1, strings, field2, ints) = struct_array_values(256);
- c.bench_function("struct_array_from_vec 256", |b| {
- b.iter(|| struct_array_from_vec(&field1, &strings, &field2, &ints))
- });
-
- let (field1, strings, field2, ints) = struct_array_values(512);
- c.bench_function("struct_array_from_vec 512", |b| {
- b.iter(|| struct_array_from_vec(&field1, &strings, &field2, &ints))
- });
-
- let (field1, strings, field2, ints) = struct_array_values(1024);
- c.bench_function("struct_array_from_vec 1024", |b| {
- b.iter(|| struct_array_from_vec(&field1, &strings, &field2, &ints))
- });
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);
diff --git a/rust/arrow/benches/array_slice.rs b/rust/arrow/benches/array_slice.rs
deleted file mode 100644
index a535c80..0000000
--- a/rust/arrow/benches/array_slice.rs
+++ /dev/null
@@ -1,52 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#[macro_use]
-extern crate criterion;
-use criterion::Criterion;
-
-extern crate arrow;
-
-use arrow::array::*;
-use std::sync::Arc;
-
-fn create_array_slice(array: &ArrayRef, length: usize) -> ArrayRef {
- array.slice(0, length)
-}
-
-fn create_array_with_nulls(size: usize) -> ArrayRef {
- let array: Float64Array = (0..size)
- .map(|i| if i % 2 == 0 { Some(1.0) } else { None })
- .collect();
- Arc::new(array)
-}
-
-fn array_slice_benchmark(c: &mut Criterion) {
- let array = create_array_with_nulls(4096);
- c.bench_function("array_slice 128", |b| {
- b.iter(|| create_array_slice(&array, 128))
- });
- c.bench_function("array_slice 512", |b| {
- b.iter(|| create_array_slice(&array, 512))
- });
- c.bench_function("array_slice 2048", |b| {
- b.iter(|| create_array_slice(&array, 2048))
- });
-}
-
-criterion_group!(benches, array_slice_benchmark);
-criterion_main!(benches);
diff --git a/rust/arrow/benches/bit_length_kernel.rs b/rust/arrow/benches/bit_length_kernel.rs
deleted file mode 100644
index 51d3134..0000000
--- a/rust/arrow/benches/bit_length_kernel.rs
+++ /dev/null
@@ -1,46 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#[macro_use]
-extern crate criterion;
-use criterion::Criterion;
-
-extern crate arrow;
-
-use arrow::{array::*, compute::kernels::length::bit_length};
-
-fn bench_bit_length(array: &StringArray) {
- criterion::black_box(bit_length(array).unwrap());
-}
-
-fn add_benchmark(c: &mut Criterion) {
- fn double_vec<T: Clone>(v: Vec<T>) -> Vec<T> {
- [&v[..], &v[..]].concat()
- }
-
- // double ["hello", " ", "world", "!"] 10 times
- let mut values = vec!["one", "on", "o", ""];
- for _ in 0..10 {
- values = double_vec(values);
- }
- let array = StringArray::from(values);
-
- c.bench_function("bit_length", |b| b.iter(|| bench_bit_length(&array)));
-}
-
-criterion_group!(benches, add_benchmark);
-criterion_main!(benches);
diff --git a/rust/arrow/benches/boolean_kernels.rs b/rust/arrow/benches/boolean_kernels.rs
deleted file mode 100644
index 6559c4e..0000000
--- a/rust/arrow/benches/boolean_kernels.rs
+++ /dev/null
@@ -1,51 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#[macro_use]
-extern crate criterion;
-use criterion::Criterion;
-
-use arrow::util::bench_util::create_boolean_array;
-
-extern crate arrow;
-
-use arrow::array::*;
-use arrow::compute::kernels::boolean as boolean_kernels;
-
-fn bench_and(lhs: &BooleanArray, rhs: &BooleanArray) {
- criterion::black_box(boolean_kernels::and(lhs, rhs).unwrap());
-}
-
-fn bench_or(lhs: &BooleanArray, rhs: &BooleanArray) {
- criterion::black_box(boolean_kernels::or(lhs, rhs).unwrap());
-}
-
-fn bench_not(array: &BooleanArray) {
- criterion::black_box(boolean_kernels::not(&array).unwrap());
-}
-
-fn add_benchmark(c: &mut Criterion) {
- let size = 2usize.pow(15);
- let array1 = create_boolean_array(size, 0.0, 0.5);
- let array2 = create_boolean_array(size, 0.0, 0.5);
- c.bench_function("and", |b| b.iter(|| bench_and(&array1, &array2)));
- c.bench_function("or", |b| b.iter(|| bench_or(&array1, &array2)));
- c.bench_function("not", |b| b.iter(|| bench_not(&array1)));
-}
-
-criterion_group!(benches, add_benchmark);
-criterion_main!(benches);
diff --git a/rust/arrow/benches/buffer_bit_ops.rs b/rust/arrow/benches/buffer_bit_ops.rs
deleted file mode 100644
index 063f39c..0000000
--- a/rust/arrow/benches/buffer_bit_ops.rs
+++ /dev/null
@@ -1,59 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#[macro_use]
-extern crate criterion;
-use criterion::Criterion;
-
-extern crate arrow;
-
-use arrow::buffer::{Buffer, MutableBuffer};
-
-/// Helper function to create arrays
-fn create_buffer(size: usize) -> Buffer {
- let mut result = MutableBuffer::new(size).with_bitset(size, false);
-
- for i in 0..size {
- result.as_slice_mut()[i] = 0b01010101 << i << (i % 4);
- }
-
- result.into()
-}
-
-fn bench_buffer_and(left: &Buffer, right: &Buffer) {
- criterion::black_box((left & right).unwrap());
-}
-
-fn bench_buffer_or(left: &Buffer, right: &Buffer) {
- criterion::black_box((left | right).unwrap());
-}
-
-fn bit_ops_benchmark(c: &mut Criterion) {
- let left = create_buffer(512 * 10);
- let right = create_buffer(512 * 10);
-
- c.bench_function("buffer_bit_ops and", |b| {
- b.iter(|| bench_buffer_and(&left, &right))
- });
-
- c.bench_function("buffer_bit_ops or", |b| {
- b.iter(|| bench_buffer_or(&left, &right))
- });
-}
-
-criterion_group!(benches, bit_ops_benchmark);
-criterion_main!(benches);
diff --git a/rust/arrow/benches/buffer_create.rs b/rust/arrow/benches/buffer_create.rs
deleted file mode 100644
index d628e03..0000000
--- a/rust/arrow/benches/buffer_create.rs
+++ /dev/null
@@ -1,190 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#[macro_use]
-extern crate criterion;
-use arrow::util::test_util::seedable_rng;
-use criterion::Criterion;
-use rand::distributions::Uniform;
-use rand::Rng;
-
-extern crate arrow;
-
-use arrow::{
- buffer::{Buffer, MutableBuffer},
- datatypes::ToByteSlice,
-};
-
-fn mutable_buffer_from_iter(data: &[Vec<bool>]) -> Vec<Buffer> {
- criterion::black_box(
- data.iter()
- .map(|vec| vec.iter().copied().collect::<MutableBuffer>().into())
- .collect::<Vec<_>>(),
- )
-}
-
-fn buffer_from_iter(data: &[Vec<bool>]) -> Vec<Buffer> {
- criterion::black_box(
- data.iter()
- .map(|vec| vec.iter().copied().collect::<Buffer>())
- .collect::<Vec<_>>(),
- )
-}
-
-fn mutable_buffer_iter_bitset(data: &[Vec<bool>]) -> Vec<Buffer> {
- criterion::black_box({
- data.iter()
- .map(|datum| {
- let mut result = MutableBuffer::new((data.len() + 7) / 8)
- .with_bitset(datum.len(), false);
- for (i, value) in datum.iter().enumerate() {
- if *value {
- unsafe {
- arrow::util::bit_util::set_bit_raw(result.as_mut_ptr(), i);
- }
- }
- }
- result.into()
- })
- .collect::<Vec<_>>()
- })
-}
-
-fn mutable_iter_extend_from_slice(data: &[Vec<u32>], capacity: usize) -> Buffer {
- criterion::black_box({
- let mut result = MutableBuffer::new(capacity);
-
- data.iter().for_each(|vec| {
- vec.iter()
- .for_each(|elem| result.extend_from_slice(elem.to_byte_slice()))
- });
-
- result.into()
- })
-}
-
-fn mutable_buffer(data: &[Vec<u32>], capacity: usize) -> Buffer {
- criterion::black_box({
- let mut result = MutableBuffer::new(capacity);
-
- data.iter().for_each(|vec| result.extend_from_slice(vec));
-
- result.into()
- })
-}
-
-fn mutable_buffer_extend(data: &[Vec<u32>], capacity: usize) -> Buffer {
- criterion::black_box({
- let mut result = MutableBuffer::new(capacity);
-
- data.iter()
- .for_each(|vec| result.extend(vec.iter().copied()));
-
- result.into()
- })
-}
-
-fn from_slice(data: &[Vec<u32>], capacity: usize) -> Buffer {
- criterion::black_box({
- let mut a = Vec::<u32>::with_capacity(capacity);
-
- data.iter().for_each(|vec| a.extend(vec));
-
- Buffer::from(a.to_byte_slice())
- })
-}
-
-fn create_data(size: usize) -> Vec<Vec<u32>> {
- let rng = &mut seedable_rng();
- let range = Uniform::new(0, 33);
-
- (0..size)
- .map(|_| {
- let size = rng.sample(range);
- seedable_rng()
- .sample_iter(&range)
- .take(size as usize)
- .collect()
- })
- .collect()
-}
-
-fn create_data_bool(size: usize) -> Vec<Vec<bool>> {
- let rng = &mut seedable_rng();
- let range = Uniform::new(0, 33);
-
- (0..size)
- .map(|_| {
- let size = rng.sample(range);
- seedable_rng()
- .sample_iter(&range)
- .take(size as usize)
- .map(|x| x > 15)
- .collect()
- })
- .collect()
-}
-fn benchmark(c: &mut Criterion) {
- let size = 2usize.pow(15);
- let data = create_data(size);
-
- let bool_data = create_data_bool(size);
- let cap = data.iter().map(|i| i.len()).sum();
- let byte_cap = cap * std::mem::size_of::<u32>();
-
- c.bench_function("mutable iter extend_from_slice", |b| {
- b.iter(|| {
- mutable_iter_extend_from_slice(
- criterion::black_box(&data),
- criterion::black_box(0),
- )
- })
- });
- c.bench_function("mutable", |b| {
- b.iter(|| mutable_buffer(criterion::black_box(&data), criterion::black_box(0)))
- });
-
- c.bench_function("mutable extend", |b| {
- b.iter(|| mutable_buffer_extend(&data, 0))
- });
-
- c.bench_function("mutable prepared", |b| {
- b.iter(|| {
- mutable_buffer(criterion::black_box(&data), criterion::black_box(byte_cap))
- })
- });
-
- c.bench_function("from_slice", |b| {
- b.iter(|| from_slice(criterion::black_box(&data), criterion::black_box(0)))
- });
- c.bench_function("from_slice prepared", |b| {
- b.iter(|| from_slice(criterion::black_box(&data), criterion::black_box(cap)))
- });
-
- c.bench_function("MutableBuffer iter bitset", |b| {
- b.iter(|| mutable_buffer_iter_bitset(criterion::black_box(&bool_data)))
- });
- c.bench_function("MutableBuffer::from_iter bool", |b| {
- b.iter(|| mutable_buffer_from_iter(criterion::black_box(&bool_data)))
- });
- c.bench_function("Buffer::from_iter bool", |b| {
- b.iter(|| buffer_from_iter(criterion::black_box(&bool_data)))
- });
-}
-
-criterion_group!(benches, benchmark);
-criterion_main!(benches);
diff --git a/rust/arrow/benches/builder.rs b/rust/arrow/benches/builder.rs
deleted file mode 100644
index fd9f319..0000000
--- a/rust/arrow/benches/builder.rs
+++ /dev/null
@@ -1,116 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-extern crate arrow;
-extern crate criterion;
-extern crate rand;
-
-use std::mem::size_of;
-
-use criterion::*;
-use rand::distributions::Standard;
-
-use arrow::array::*;
-use arrow::util::test_util::seedable_rng;
-use rand::Rng;
-
-// Build arrays with 512k elements.
-const BATCH_SIZE: usize = 8 << 10;
-const NUM_BATCHES: usize = 64;
-
-fn bench_primitive(c: &mut Criterion) {
- let data: [i64; BATCH_SIZE] = [100; BATCH_SIZE];
-
- let mut group = c.benchmark_group("bench_primitive");
- group.throughput(Throughput::Bytes(
- ((data.len() * NUM_BATCHES * size_of::<i64>()) as u32).into(),
- ));
- group.bench_function("bench_primitive", |b| {
- b.iter(|| {
- let mut builder = Int64Builder::new(64);
- for _ in 0..NUM_BATCHES {
- let _ = black_box(builder.append_slice(&data[..]));
- }
- black_box(builder.finish());
- })
- });
- group.finish();
-}
-
-fn bench_primitive_nulls(c: &mut Criterion) {
- let mut group = c.benchmark_group("bench_primitive_nulls");
- group.bench_function("bench_primitive_nulls", |b| {
- b.iter(|| {
- let mut builder = UInt8Builder::new(64);
- for _ in 0..NUM_BATCHES * BATCH_SIZE {
- let _ = black_box(builder.append_null());
- }
- black_box(builder.finish());
- })
- });
- group.finish();
-}
-
-fn bench_bool(c: &mut Criterion) {
- let data: Vec<bool> = seedable_rng()
- .sample_iter(&Standard)
- .take(BATCH_SIZE)
- .collect();
- let data_len = data.len();
-
- let mut group = c.benchmark_group("bench_bool");
- group.throughput(Throughput::Bytes(
- ((data_len * NUM_BATCHES * size_of::<bool>()) as u32).into(),
- ));
- group.bench_function("bench_bool", |b| {
- b.iter(|| {
- let mut builder = BooleanBuilder::new(64);
- for _ in 0..NUM_BATCHES {
- let _ = black_box(builder.append_slice(&data[..]));
- }
- black_box(builder.finish());
- })
- });
- group.finish();
-}
-
-fn bench_string(c: &mut Criterion) {
- const SAMPLE_STRING: &str = "sample string";
- let mut group = c.benchmark_group("bench_primitive");
- group.throughput(Throughput::Bytes(
- ((BATCH_SIZE * NUM_BATCHES * SAMPLE_STRING.len()) as u32).into(),
- ));
- group.bench_function("bench_string", |b| {
- b.iter(|| {
- let mut builder = StringBuilder::new(64);
- for _ in 0..NUM_BATCHES * BATCH_SIZE {
- let _ = black_box(builder.append_value(SAMPLE_STRING));
- }
- black_box(builder.finish());
- })
- });
- group.finish();
-}
-
-criterion_group!(
- benches,
- bench_primitive,
- bench_primitive_nulls,
- bench_bool,
- bench_string
-);
-criterion_main!(benches);
diff --git a/rust/arrow/benches/cast_kernels.rs b/rust/arrow/benches/cast_kernels.rs
deleted file mode 100644
index d164e1f..0000000
--- a/rust/arrow/benches/cast_kernels.rs
+++ /dev/null
@@ -1,185 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#[macro_use]
-extern crate criterion;
-use criterion::Criterion;
-use rand::distributions::{Distribution, Standard, Uniform};
-use rand::Rng;
-
-use std::sync::Arc;
-
-extern crate arrow;
-
-use arrow::array::*;
-use arrow::compute::cast;
-use arrow::datatypes::*;
-use arrow::util::bench_util::*;
-use arrow::util::test_util::seedable_rng;
-
-fn build_array<T: ArrowPrimitiveType>(size: usize) -> ArrayRef
-where
- Standard: Distribution<T::Native>,
-{
- let array = create_primitive_array::<T>(size, 0.1);
- Arc::new(array)
-}
-
-fn build_utf8_date_array(size: usize, with_nulls: bool) -> ArrayRef {
- use chrono::NaiveDate;
-
- // use random numbers to avoid spurious compiler optimizations wrt to branching
- let mut rng = seedable_rng();
- let mut builder = StringBuilder::new(size);
- let range = Uniform::new(0, 737776);
-
- for _ in 0..size {
- if with_nulls && rng.gen::<f32>() > 0.8 {
- builder.append_null().unwrap();
- } else {
- let string = NaiveDate::from_num_days_from_ce(rng.sample(range))
- .format("%Y-%m-%d")
- .to_string();
- builder.append_value(&string).unwrap();
- }
- }
- Arc::new(builder.finish())
-}
-
-fn build_utf8_date_time_array(size: usize, with_nulls: bool) -> ArrayRef {
- use chrono::NaiveDateTime;
-
- // use random numbers to avoid spurious compiler optimizations wrt to branching
- let mut rng = seedable_rng();
- let mut builder = StringBuilder::new(size);
- let range = Uniform::new(0, 1608071414123);
-
- for _ in 0..size {
- if with_nulls && rng.gen::<f32>() > 0.8 {
- builder.append_null().unwrap();
- } else {
- let string = NaiveDateTime::from_timestamp(rng.sample(range), 0)
- .format("%Y-%m-%dT%H:%M:%S")
- .to_string();
- builder.append_value(&string).unwrap();
- }
- }
- Arc::new(builder.finish())
-}
-
-// cast array from specified primitive array type to desired data type
-fn cast_array(array: &ArrayRef, to_type: DataType) {
- criterion::black_box(cast(array, &to_type).unwrap());
-}
-
-fn add_benchmark(c: &mut Criterion) {
- let i32_array = build_array::<Int32Type>(512);
- let i64_array = build_array::<Int64Type>(512);
- let f32_array = build_array::<Float32Type>(512);
- let f32_utf8_array = cast(&build_array::<Float32Type>(512), &DataType::Utf8).unwrap();
-
- let f64_array = build_array::<Float64Type>(512);
- let date64_array = build_array::<Date64Type>(512);
- let date32_array = build_array::<Date32Type>(512);
- let time32s_array = build_array::<Time32SecondType>(512);
- let time64ns_array = build_array::<Time64NanosecondType>(512);
- let time_ns_array = build_array::<TimestampNanosecondType>(512);
- let time_ms_array = build_array::<TimestampMillisecondType>(512);
- let utf8_date_array = build_utf8_date_array(512, true);
- let utf8_date_time_array = build_utf8_date_time_array(512, true);
-
- c.bench_function("cast int32 to int32 512", |b| {
- b.iter(|| cast_array(&i32_array, DataType::Int32))
- });
- c.bench_function("cast int32 to uint32 512", |b| {
- b.iter(|| cast_array(&i32_array, DataType::UInt32))
- });
- c.bench_function("cast int32 to float32 512", |b| {
- b.iter(|| cast_array(&i32_array, DataType::Float32))
- });
- c.bench_function("cast int32 to float64 512", |b| {
- b.iter(|| cast_array(&i32_array, DataType::Float64))
- });
- c.bench_function("cast int32 to int64 512", |b| {
- b.iter(|| cast_array(&i32_array, DataType::Int64))
- });
- c.bench_function("cast float32 to int32 512", |b| {
- b.iter(|| cast_array(&f32_array, DataType::Int32))
- });
- c.bench_function("cast float64 to float32 512", |b| {
- b.iter(|| cast_array(&f64_array, DataType::Float32))
- });
- c.bench_function("cast float64 to uint64 512", |b| {
- b.iter(|| cast_array(&f64_array, DataType::UInt64))
- });
- c.bench_function("cast int64 to int32 512", |b| {
- b.iter(|| cast_array(&i64_array, DataType::Int32))
- });
- c.bench_function("cast date64 to date32 512", |b| {
- b.iter(|| cast_array(&date64_array, DataType::Date32))
- });
- c.bench_function("cast date32 to date64 512", |b| {
- b.iter(|| cast_array(&date32_array, DataType::Date64))
- });
- c.bench_function("cast time32s to time32ms 512", |b| {
- b.iter(|| cast_array(&time32s_array, DataType::Time32(TimeUnit::Millisecond)))
- });
- c.bench_function("cast time32s to time64us 512", |b| {
- b.iter(|| cast_array(&time32s_array, DataType::Time64(TimeUnit::Microsecond)))
- });
- c.bench_function("cast time64ns to time32s 512", |b| {
- b.iter(|| cast_array(&time64ns_array, DataType::Time32(TimeUnit::Second)))
- });
- c.bench_function("cast timestamp_ns to timestamp_s 512", |b| {
- b.iter(|| {
- cast_array(
- &time_ns_array,
- DataType::Timestamp(TimeUnit::Nanosecond, None),
- )
- })
- });
- c.bench_function("cast timestamp_ms to timestamp_ns 512", |b| {
- b.iter(|| {
- cast_array(
- &time_ms_array,
- DataType::Timestamp(TimeUnit::Nanosecond, None),
- )
- })
- });
- c.bench_function("cast utf8 to f32", |b| {
- b.iter(|| cast_array(&f32_utf8_array, DataType::Float32))
- });
- c.bench_function("cast i64 to string 512", |b| {
- b.iter(|| cast_array(&i64_array, DataType::Utf8))
- });
- c.bench_function("cast f32 to string 512", |b| {
- b.iter(|| cast_array(&f32_array, DataType::Utf8))
- });
-
- c.bench_function("cast timestamp_ms to i64 512", |b| {
- b.iter(|| cast_array(&time_ms_array, DataType::Int64))
- });
- c.bench_function("cast utf8 to date32 512", |b| {
- b.iter(|| cast_array(&utf8_date_array, DataType::Date32))
- });
- c.bench_function("cast utf8 to date64 512", |b| {
- b.iter(|| cast_array(&utf8_date_time_array, DataType::Date64))
- });
-}
-
-criterion_group!(benches, add_benchmark);
-criterion_main!(benches);
diff --git a/rust/arrow/benches/comparison_kernels.rs b/rust/arrow/benches/comparison_kernels.rs
deleted file mode 100644
index a3df556..0000000
--- a/rust/arrow/benches/comparison_kernels.rs
+++ /dev/null
@@ -1,201 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#[macro_use]
-extern crate criterion;
-use criterion::Criterion;
-
-extern crate arrow;
-
-use arrow::compute::*;
-use arrow::datatypes::ArrowNumericType;
-use arrow::util::bench_util::*;
-use arrow::{array::*, datatypes::Float32Type};
-
-fn bench_eq<T>(arr_a: &PrimitiveArray<T>, arr_b: &PrimitiveArray<T>)
-where
- T: ArrowNumericType,
-{
- eq(criterion::black_box(arr_a), criterion::black_box(arr_b)).unwrap();
-}
-
-fn bench_eq_scalar<T>(arr_a: &PrimitiveArray<T>, value_b: T::Native)
-where
- T: ArrowNumericType,
-{
- eq_scalar(criterion::black_box(arr_a), criterion::black_box(value_b)).unwrap();
-}
-
-fn bench_neq<T>(arr_a: &PrimitiveArray<T>, arr_b: &PrimitiveArray<T>)
-where
- T: ArrowNumericType,
-{
- neq(criterion::black_box(arr_a), criterion::black_box(arr_b)).unwrap();
-}
-
-fn bench_neq_scalar<T>(arr_a: &PrimitiveArray<T>, value_b: T::Native)
-where
- T: ArrowNumericType,
-{
- neq_scalar(criterion::black_box(arr_a), criterion::black_box(value_b)).unwrap();
-}
-
-fn bench_lt<T>(arr_a: &PrimitiveArray<T>, arr_b: &PrimitiveArray<T>)
-where
- T: ArrowNumericType,
-{
- lt(criterion::black_box(arr_a), criterion::black_box(arr_b)).unwrap();
-}
-
-fn bench_lt_scalar<T>(arr_a: &PrimitiveArray<T>, value_b: T::Native)
-where
- T: ArrowNumericType,
-{
- lt_scalar(criterion::black_box(arr_a), criterion::black_box(value_b)).unwrap();
-}
-
-fn bench_lt_eq<T>(arr_a: &PrimitiveArray<T>, arr_b: &PrimitiveArray<T>)
-where
- T: ArrowNumericType,
-{
- lt_eq(criterion::black_box(arr_a), criterion::black_box(arr_b)).unwrap();
-}
-
-fn bench_lt_eq_scalar<T>(arr_a: &PrimitiveArray<T>, value_b: T::Native)
-where
- T: ArrowNumericType,
-{
- lt_eq_scalar(criterion::black_box(arr_a), criterion::black_box(value_b)).unwrap();
-}
-
-fn bench_gt<T>(arr_a: &PrimitiveArray<T>, arr_b: &PrimitiveArray<T>)
-where
- T: ArrowNumericType,
-{
- gt(criterion::black_box(arr_a), criterion::black_box(arr_b)).unwrap();
-}
-
-fn bench_gt_scalar<T>(arr_a: &PrimitiveArray<T>, value_b: T::Native)
-where
- T: ArrowNumericType,
-{
- gt_scalar(criterion::black_box(arr_a), criterion::black_box(value_b)).unwrap();
-}
-
-fn bench_gt_eq<T>(arr_a: &PrimitiveArray<T>, arr_b: &PrimitiveArray<T>)
-where
- T: ArrowNumericType,
-{
- gt_eq(criterion::black_box(arr_a), criterion::black_box(arr_b)).unwrap();
-}
-
-fn bench_gt_eq_scalar<T>(arr_a: &PrimitiveArray<T>, value_b: T::Native)
-where
- T: ArrowNumericType,
-{
- gt_eq_scalar(criterion::black_box(arr_a), criterion::black_box(value_b)).unwrap();
-}
-
-fn bench_like_utf8_scalar(arr_a: &StringArray, value_b: &str) {
- like_utf8_scalar(criterion::black_box(arr_a), criterion::black_box(value_b)).unwrap();
-}
-
-fn bench_nlike_utf8_scalar(arr_a: &StringArray, value_b: &str) {
- nlike_utf8_scalar(criterion::black_box(arr_a), criterion::black_box(value_b))
- .unwrap();
-}
-
-fn add_benchmark(c: &mut Criterion) {
- let size = 65536;
- let arr_a = create_primitive_array_with_seed::<Float32Type>(size, 0.0, 42);
- let arr_b = create_primitive_array_with_seed::<Float32Type>(size, 0.0, 43);
-
- let arr_string = create_string_array::<i32>(size, 0.0);
-
- c.bench_function("eq Float32", |b| b.iter(|| bench_eq(&arr_a, &arr_b)));
- c.bench_function("eq scalar Float32", |b| {
- b.iter(|| bench_eq_scalar(&arr_a, 1.0))
- });
-
- c.bench_function("neq Float32", |b| b.iter(|| bench_neq(&arr_a, &arr_b)));
- c.bench_function("neq scalar Float32", |b| {
- b.iter(|| bench_neq_scalar(&arr_a, 1.0))
- });
-
- c.bench_function("lt Float32", |b| b.iter(|| bench_lt(&arr_a, &arr_b)));
- c.bench_function("lt scalar Float32", |b| {
- b.iter(|| bench_lt_scalar(&arr_a, 1.0))
- });
-
- c.bench_function("lt_eq Float32", |b| b.iter(|| bench_lt_eq(&arr_a, &arr_b)));
- c.bench_function("lt_eq scalar Float32", |b| {
- b.iter(|| bench_lt_eq_scalar(&arr_a, 1.0))
- });
-
- c.bench_function("gt Float32", |b| b.iter(|| bench_gt(&arr_a, &arr_b)));
- c.bench_function("gt scalar Float32", |b| {
- b.iter(|| bench_gt_scalar(&arr_a, 1.0))
- });
-
- c.bench_function("gt_eq Float32", |b| b.iter(|| bench_gt_eq(&arr_a, &arr_b)));
- c.bench_function("gt_eq scalar Float32", |b| {
- b.iter(|| bench_gt_eq_scalar(&arr_a, 1.0))
- });
-
- c.bench_function("like_utf8 scalar equals", |b| {
- b.iter(|| bench_like_utf8_scalar(&arr_string, "xxxx"))
- });
-
- c.bench_function("like_utf8 scalar contains", |b| {
- b.iter(|| bench_like_utf8_scalar(&arr_string, "%xxxx%"))
- });
-
- c.bench_function("like_utf8 scalar ends with", |b| {
- b.iter(|| bench_like_utf8_scalar(&arr_string, "xxxx%"))
- });
-
- c.bench_function("like_utf8 scalar starts with", |b| {
- b.iter(|| bench_like_utf8_scalar(&arr_string, "%xxxx"))
- });
-
- c.bench_function("like_utf8 scalar complex", |b| {
- b.iter(|| bench_like_utf8_scalar(&arr_string, "%xx_xx%xxx"))
- });
-
- c.bench_function("nlike_utf8 scalar equals", |b| {
- b.iter(|| bench_nlike_utf8_scalar(&arr_string, "xxxx"))
- });
-
- c.bench_function("nlike_utf8 scalar contains", |b| {
- b.iter(|| bench_nlike_utf8_scalar(&arr_string, "%xxxx%"))
- });
-
- c.bench_function("nlike_utf8 scalar ends with", |b| {
- b.iter(|| bench_nlike_utf8_scalar(&arr_string, "xxxx%"))
- });
-
- c.bench_function("nlike_utf8 scalar starts with", |b| {
- b.iter(|| bench_nlike_utf8_scalar(&arr_string, "%xxxx"))
- });
-
- c.bench_function("nlike_utf8 scalar complex", |b| {
- b.iter(|| bench_nlike_utf8_scalar(&arr_string, "%xx_xx%xxx"))
- });
-}
-
-criterion_group!(benches, add_benchmark);
-criterion_main!(benches);
diff --git a/rust/arrow/benches/concatenate_kernel.rs b/rust/arrow/benches/concatenate_kernel.rs
deleted file mode 100644
index 3fff2ab..0000000
--- a/rust/arrow/benches/concatenate_kernel.rs
+++ /dev/null
@@ -1,66 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#[macro_use]
-extern crate criterion;
-use criterion::Criterion;
-
-extern crate arrow;
-
-use arrow::array::*;
-use arrow::compute::concat;
-use arrow::datatypes::*;
-use arrow::util::bench_util::*;
-
-fn bench_concat(v1: &dyn Array, v2: &dyn Array) {
- criterion::black_box(concat(&[v1, v2]).unwrap());
-}
-
-fn bench_concat_arrays(arrays: &[&dyn Array]) {
- criterion::black_box(concat(arrays).unwrap());
-}
-
-fn add_benchmark(c: &mut Criterion) {
- let v1 = create_primitive_array::<Int32Type>(1024, 0.0);
- let v2 = create_primitive_array::<Int32Type>(1024, 0.0);
- c.bench_function("concat i32 1024", |b| b.iter(|| bench_concat(&v1, &v2)));
-
- let v1 = create_primitive_array::<Int32Type>(1024, 0.5);
- let v2 = create_primitive_array::<Int32Type>(1024, 0.5);
- c.bench_function("concat i32 nulls 1024", |b| {
- b.iter(|| bench_concat(&v1, &v2))
- });
-
- let small_array = create_primitive_array::<Int32Type>(4, 0.0);
- let arrays: Vec<_> = (0..1024).map(|_| &small_array as &dyn Array).collect();
- c.bench_function("concat 1024 arrays i32 4", |b| {
- b.iter(|| bench_concat_arrays(&arrays))
- });
-
- let v1 = create_string_array::<i32>(1024, 0.0);
- let v2 = create_string_array::<i32>(1024, 0.0);
- c.bench_function("concat str 1024", |b| b.iter(|| bench_concat(&v1, &v2)));
-
- let v1 = create_string_array::<i32>(1024, 0.5);
- let v2 = create_string_array::<i32>(1024, 0.5);
- c.bench_function("concat str nulls 1024", |b| {
- b.iter(|| bench_concat(&v1, &v2))
- });
-}
-
-criterion_group!(benches, add_benchmark);
-criterion_main!(benches);
diff --git a/rust/arrow/benches/csv_writer.rs b/rust/arrow/benches/csv_writer.rs
deleted file mode 100644
index 9b01853..0000000
--- a/rust/arrow/benches/csv_writer.rs
+++ /dev/null
@@ -1,70 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-extern crate arrow;
-extern crate criterion;
-
-use criterion::*;
-
-use arrow::array::*;
-use arrow::csv;
-use arrow::datatypes::*;
-use arrow::record_batch::RecordBatch;
-use std::fs::File;
-use std::sync::Arc;
-
-fn record_batches_to_csv() {
- let schema = Schema::new(vec![
- Field::new("c1", DataType::Utf8, false),
- Field::new("c2", DataType::Float64, true),
- Field::new("c3", DataType::UInt32, false),
- Field::new("c3", DataType::Boolean, true),
- ]);
-
- let c1 = StringArray::from(vec![
- "Lorem ipsum dolor sit amet",
- "consectetur adipiscing elit",
- "sed do eiusmod tempor",
- ]);
- let c2 = PrimitiveArray::<Float64Type>::from(vec![
- Some(123.564532),
- None,
- Some(-556132.25),
- ]);
- let c3 = PrimitiveArray::<UInt32Type>::from(vec![3, 2, 1]);
- let c4 = BooleanArray::from(vec![Some(true), Some(false), None]);
-
- let b = RecordBatch::try_new(
- Arc::new(schema),
- vec![Arc::new(c1), Arc::new(c2), Arc::new(c3), Arc::new(c4)],
- )
- .unwrap();
- let file = File::create("target/bench_write_csv.csv").unwrap();
- let mut writer = csv::Writer::new(file);
- let batches = vec![&b, &b, &b, &b, &b, &b, &b, &b, &b, &b, &b];
- #[allow(clippy::unit_arg)]
- criterion::black_box(for batch in batches {
- writer.write(batch).unwrap()
- });
-}
-
-fn criterion_benchmark(c: &mut Criterion) {
- c.bench_function("record_batches_to_csv", |b| b.iter(record_batches_to_csv));
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);
diff --git a/rust/arrow/benches/equal.rs b/rust/arrow/benches/equal.rs
deleted file mode 100644
index af53550..0000000
--- a/rust/arrow/benches/equal.rs
+++ /dev/null
@@ -1,57 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// Allowed because we use `arr == arr` in benchmarks
-#![allow(clippy::eq_op)]
-
-#[macro_use]
-extern crate criterion;
-use criterion::Criterion;
-
-extern crate arrow;
-
-use arrow::util::bench_util::*;
-use arrow::{array::*, datatypes::Float32Type};
-
-fn bench_equal<A: Array + PartialEq<A>>(arr_a: &A) {
- criterion::black_box(arr_a == arr_a);
-}
-
-fn add_benchmark(c: &mut Criterion) {
- let arr_a = create_primitive_array::<Float32Type>(512, 0.0);
- c.bench_function("equal_512", |b| b.iter(|| bench_equal(&arr_a)));
-
- let arr_a_nulls = create_primitive_array::<Float32Type>(512, 0.5);
- c.bench_function("equal_nulls_512", |b| b.iter(|| bench_equal(&arr_a_nulls)));
-
- let arr_a = create_string_array::<i32>(512, 0.0);
- c.bench_function("equal_string_512", |b| b.iter(|| bench_equal(&arr_a)));
-
- let arr_a_nulls = create_string_array::<i32>(512, 0.5);
- c.bench_function("equal_string_nulls_512", |b| {
- b.iter(|| bench_equal(&arr_a_nulls))
- });
-
- let arr_a = create_boolean_array(512, 0.0, 0.5);
- c.bench_function("equal_bool_512", |b| b.iter(|| bench_equal(&arr_a)));
-
- let arr_a = create_boolean_array(513, 0.0, 0.5);
- c.bench_function("equal_bool_513", |b| b.iter(|| bench_equal(&arr_a)));
-}
-
-criterion_group!(benches, add_benchmark);
-criterion_main!(benches);
diff --git a/rust/arrow/benches/filter_kernels.rs b/rust/arrow/benches/filter_kernels.rs
deleted file mode 100644
index ca317b4..0000000
--- a/rust/arrow/benches/filter_kernels.rs
+++ /dev/null
@@ -1,106 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-extern crate arrow;
-
-use arrow::compute::Filter;
-use arrow::util::bench_util::*;
-
-use arrow::array::*;
-use arrow::compute::{build_filter, filter};
-use arrow::datatypes::{Float32Type, UInt8Type};
-
-use criterion::{criterion_group, criterion_main, Criterion};
-
-fn bench_filter(data_array: &dyn Array, filter_array: &BooleanArray) {
- criterion::black_box(filter(data_array, filter_array).unwrap());
-}
-
-fn bench_built_filter<'a>(filter: &Filter<'a>, data: &impl Array) {
- criterion::black_box(filter(&data.data()));
-}
-
-fn add_benchmark(c: &mut Criterion) {
- let size = 65536;
- let filter_array = create_boolean_array(size, 0.0, 0.5);
- let dense_filter_array = create_boolean_array(size, 0.0, 1.0 - 1.0 / 1024.0);
- let sparse_filter_array = create_boolean_array(size, 0.0, 1.0 / 1024.0);
-
- let filter = build_filter(&filter_array).unwrap();
- let dense_filter = build_filter(&dense_filter_array).unwrap();
- let sparse_filter = build_filter(&sparse_filter_array).unwrap();
-
- let data_array = create_primitive_array::<UInt8Type>(size, 0.0);
-
- c.bench_function("filter u8", |b| {
- b.iter(|| bench_filter(&data_array, &filter_array))
- });
- c.bench_function("filter u8 high selectivity", |b| {
- b.iter(|| bench_filter(&data_array, &dense_filter_array))
- });
- c.bench_function("filter u8 low selectivity", |b| {
- b.iter(|| bench_filter(&data_array, &sparse_filter_array))
- });
-
- c.bench_function("filter context u8", |b| {
- b.iter(|| bench_built_filter(&filter, &data_array))
- });
- c.bench_function("filter context u8 high selectivity", |b| {
- b.iter(|| bench_built_filter(&dense_filter, &data_array))
- });
- c.bench_function("filter context u8 low selectivity", |b| {
- b.iter(|| bench_built_filter(&sparse_filter, &data_array))
- });
-
- let data_array = create_primitive_array::<UInt8Type>(size, 0.5);
- c.bench_function("filter context u8 w NULLs", |b| {
- b.iter(|| bench_built_filter(&filter, &data_array))
- });
- c.bench_function("filter context u8 w NULLs high selectivity", |b| {
- b.iter(|| bench_built_filter(&dense_filter, &data_array))
- });
- c.bench_function("filter context u8 w NULLs low selectivity", |b| {
- b.iter(|| bench_built_filter(&sparse_filter, &data_array))
- });
-
- let data_array = create_primitive_array::<Float32Type>(size, 0.5);
- c.bench_function("filter f32", |b| {
- b.iter(|| bench_filter(&data_array, &filter_array))
- });
- c.bench_function("filter context f32", |b| {
- b.iter(|| bench_built_filter(&filter, &data_array))
- });
- c.bench_function("filter context f32 high selectivity", |b| {
- b.iter(|| bench_built_filter(&dense_filter, &data_array))
- });
- c.bench_function("filter context f32 low selectivity", |b| {
- b.iter(|| bench_built_filter(&sparse_filter, &data_array))
- });
-
- let data_array = create_string_array::<i32>(size, 0.5);
- c.bench_function("filter context string", |b| {
- b.iter(|| bench_built_filter(&filter, &data_array))
- });
- c.bench_function("filter context string high selectivity", |b| {
- b.iter(|| bench_built_filter(&dense_filter, &data_array))
- });
- c.bench_function("filter context string low selectivity", |b| {
- b.iter(|| bench_built_filter(&sparse_filter, &data_array))
- });
-}
-
-criterion_group!(benches, add_benchmark);
-criterion_main!(benches);
diff --git a/rust/arrow/benches/json_reader.rs b/rust/arrow/benches/json_reader.rs
deleted file mode 100644
index ef3ddf0..0000000
--- a/rust/arrow/benches/json_reader.rs
+++ /dev/null
@@ -1,112 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-extern crate arrow;
-extern crate criterion;
-
-use criterion::*;
-
-use arrow::datatypes::*;
-use arrow::json::ReaderBuilder;
-use std::io::Cursor;
-use std::sync::Arc;
-
-fn json_primitive_to_record_batch() {
- let schema = Arc::new(Schema::new(vec![
- Field::new("c1", DataType::Utf8, true),
- Field::new("c2", DataType::Float64, true),
- Field::new("c3", DataType::UInt32, true),
- Field::new("c4", DataType::Boolean, true),
- ]));
- let builder = ReaderBuilder::new().with_schema(schema).with_batch_size(64);
- let json_content = r#"
- {"c1": "eleven", "c2": 6.2222222225, "c3": 5.0, "c4": false}
- {"c1": "twelve", "c2": -55555555555555.2, "c3": 3}
- {"c1": null, "c2": 3, "c3": 125, "c4": null}
- {"c2": -35, "c3": 100.0, "c4": true}
- {"c1": "fifteen", "c2": null, "c4": true}
- {"c1": "eleven", "c2": 6.2222222225, "c3": 5.0, "c4": false}
- {"c1": "twelve", "c2": -55555555555555.2, "c3": 3}
- {"c1": null, "c2": 3, "c3": 125, "c4": null}
- {"c2": -35, "c3": 100.0, "c4": true}
- {"c1": "fifteen", "c2": null, "c4": true}
- "#;
- let cursor = Cursor::new(json_content);
- let mut reader = builder.build(cursor).unwrap();
- #[allow(clippy::unit_arg)]
- criterion::black_box({
- reader.next().unwrap();
- });
-}
-
-fn json_list_primitive_to_record_batch() {
- let schema = Arc::new(Schema::new(vec![
- Field::new(
- "c1",
- DataType::List(Box::new(Field::new("item", DataType::Utf8, true))),
- true,
- ),
- Field::new(
- "c2",
- DataType::List(Box::new(Field::new("item", DataType::Float64, true))),
- true,
- ),
- Field::new(
- "c3",
- DataType::List(Box::new(Field::new("item", DataType::UInt32, true))),
- true,
- ),
- Field::new(
- "c4",
- DataType::List(Box::new(Field::new("item", DataType::Boolean, true))),
- true,
- ),
- ]));
- let builder = ReaderBuilder::new().with_schema(schema).with_batch_size(64);
- let json_content = r#"
- {"c1": ["eleven"], "c2": [6.2222222225, -3.2, null], "c3": [5.0, 6], "c4": [false, true]}
- {"c1": ["twelve"], "c2": [-55555555555555.2, 12500000.0], "c3": [3, 4, 5]}
- {"c1": null, "c2": [3], "c3": [125, 127, 129], "c4": [null, false, true]}
- {"c2": [-35], "c3": [100.0, 200.0], "c4": null}
- {"c1": ["fifteen"], "c2": [null, 2.1, 1.5, -3], "c4": [true, false, null]}
- {"c1": ["fifteen"], "c2": [], "c4": [true, false, null]}
- {"c1": ["eleven"], "c2": [6.2222222225, -3.2, null], "c3": [5.0, 6], "c4": [false, true]}
- {"c1": ["twelve"], "c2": [-55555555555555.2, 12500000.0], "c3": [3, 4, 5]}
- {"c1": null, "c2": [3], "c3": [125, 127, 129], "c4": [null, false, true]}
- {"c2": [-35], "c3": [100.0, 200.0], "c4": null}
- {"c1": ["fifteen"], "c2": [null, 2.1, 1.5, -3], "c4": [true, false, null]}
- {"c1": ["fifteen"], "c2": [], "c4": [true, false, null]}
- "#;
- let cursor = Cursor::new(json_content);
- let mut reader = builder.build(cursor).unwrap();
- #[allow(clippy::unit_arg)]
- criterion::black_box({
- reader.next().unwrap();
- });
-}
-
-fn criterion_benchmark(c: &mut Criterion) {
- c.bench_function("json_primitive_to_record_batch", |b| {
- b.iter(json_primitive_to_record_batch)
- });
- c.bench_function("json_list_primitive_to_record_batch", |b| {
- b.iter(json_list_primitive_to_record_batch)
- });
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);
diff --git a/rust/arrow/benches/length_kernel.rs b/rust/arrow/benches/length_kernel.rs
deleted file mode 100644
index b70f637..0000000
--- a/rust/arrow/benches/length_kernel.rs
+++ /dev/null
@@ -1,47 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#[macro_use]
-extern crate criterion;
-use criterion::Criterion;
-
-extern crate arrow;
-
-use arrow::array::*;
-use arrow::compute::kernels::length::length;
-
-fn bench_length(array: &StringArray) {
- criterion::black_box(length(array).unwrap());
-}
-
-fn add_benchmark(c: &mut Criterion) {
- fn double_vec<T: Clone>(v: Vec<T>) -> Vec<T> {
- [&v[..], &v[..]].concat()
- }
-
- // double ["hello", " ", "world", "!"] 10 times
- let mut values = vec!["one", "on", "o", ""];
- for _ in 0..10 {
- values = double_vec(values);
- }
- let array = StringArray::from(values);
-
- c.bench_function("length", |b| b.iter(|| bench_length(&array)));
-}
-
-criterion_group!(benches, add_benchmark);
-criterion_main!(benches);
diff --git a/rust/arrow/benches/mutable_array.rs b/rust/arrow/benches/mutable_array.rs
deleted file mode 100644
index 52da38a..0000000
--- a/rust/arrow/benches/mutable_array.rs
+++ /dev/null
@@ -1,60 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#[macro_use]
-extern crate criterion;
-use criterion::Criterion;
-
-use rand::Rng;
-
-extern crate arrow;
-
-use arrow::util::test_util::seedable_rng;
-use arrow::{array::*, util::bench_util::create_string_array};
-
-fn create_slices(size: usize) -> Vec<(usize, usize)> {
- let rng = &mut seedable_rng();
-
- (0..size)
- .map(|_| {
- let start = rng.gen_range(0, size / 2);
- let end = rng.gen_range(start + 1, size);
- (start, end)
- })
- .collect()
-}
-
-fn bench<T: Array>(v1: &T, slices: &[(usize, usize)]) {
- let mut mutable = MutableArrayData::new(vec![v1.data_ref()], false, 5);
- for (start, end) in slices {
- mutable.extend(0, *start, *end)
- }
- mutable.freeze();
-}
-
-fn add_benchmark(c: &mut Criterion) {
- let v1 = create_string_array::<i32>(1024, 0.0);
- let v2 = create_slices(1024);
- c.bench_function("mutable str 1024", |b| b.iter(|| bench(&v1, &v2)));
-
- let v1 = create_string_array::<i32>(1024, 0.5);
- let v2 = create_slices(1024);
- c.bench_function("mutable str nulls 1024", |b| b.iter(|| bench(&v1, &v2)));
-}
-
-criterion_group!(benches, add_benchmark);
-criterion_main!(benches);
diff --git a/rust/arrow/benches/sort_kernel.rs b/rust/arrow/benches/sort_kernel.rs
deleted file mode 100644
index 74dc0ce..0000000
--- a/rust/arrow/benches/sort_kernel.rs
+++ /dev/null
@@ -1,121 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#[macro_use]
-extern crate criterion;
-use criterion::Criterion;
-
-use std::sync::Arc;
-
-extern crate arrow;
-
-use arrow::compute::kernels::sort::{lexsort, SortColumn};
-use arrow::util::bench_util::*;
-use arrow::{array::*, datatypes::Float32Type};
-
-fn create_array(size: usize, with_nulls: bool) -> ArrayRef {
- let null_density = if with_nulls { 0.5 } else { 0.0 };
- let array = create_primitive_array::<Float32Type>(size, null_density);
- Arc::new(array)
-}
-
-fn bench_sort(arr_a: &ArrayRef, array_b: &ArrayRef, limit: Option<usize>) {
- let columns = vec![
- SortColumn {
- values: arr_a.clone(),
- options: None,
- },
- SortColumn {
- values: array_b.clone(),
- options: None,
- },
- ];
-
- criterion::black_box(lexsort(&columns, limit).unwrap());
-}
-
-fn add_benchmark(c: &mut Criterion) {
- let arr_a = create_array(2u64.pow(10) as usize, false);
- let arr_b = create_array(2u64.pow(10) as usize, false);
-
- c.bench_function("sort 2^10", |b| b.iter(|| bench_sort(&arr_a, &arr_b, None)));
-
- let arr_a = create_array(2u64.pow(12) as usize, false);
- let arr_b = create_array(2u64.pow(12) as usize, false);
-
- c.bench_function("sort 2^12", |b| b.iter(|| bench_sort(&arr_a, &arr_b, None)));
-
- let arr_a = create_array(2u64.pow(10) as usize, true);
- let arr_b = create_array(2u64.pow(10) as usize, true);
-
- c.bench_function("sort nulls 2^10", |b| {
- b.iter(|| bench_sort(&arr_a, &arr_b, None))
- });
-
- let arr_a = create_array(2u64.pow(12) as usize, true);
- let arr_b = create_array(2u64.pow(12) as usize, true);
-
- c.bench_function("sort nulls 2^12", |b| {
- b.iter(|| bench_sort(&arr_a, &arr_b, None))
- });
-
- // with limit
- {
- let arr_a = create_array(2u64.pow(12) as usize, false);
- let arr_b = create_array(2u64.pow(12) as usize, false);
- c.bench_function("sort 2^12 limit 10", |b| {
- b.iter(|| bench_sort(&arr_a, &arr_b, Some(10)))
- });
-
- let arr_a = create_array(2u64.pow(12) as usize, false);
- let arr_b = create_array(2u64.pow(12) as usize, false);
- c.bench_function("sort 2^12 limit 100", |b| {
- b.iter(|| bench_sort(&arr_a, &arr_b, Some(100)))
- });
-
- let arr_a = create_array(2u64.pow(12) as usize, false);
- let arr_b = create_array(2u64.pow(12) as usize, false);
- c.bench_function("sort 2^12 limit 1000", |b| {
- b.iter(|| bench_sort(&arr_a, &arr_b, Some(1000)))
- });
-
- let arr_a = create_array(2u64.pow(12) as usize, false);
- let arr_b = create_array(2u64.pow(12) as usize, false);
- c.bench_function("sort 2^12 limit 2^12", |b| {
- b.iter(|| bench_sort(&arr_a, &arr_b, Some(2u64.pow(12) as usize)))
- });
-
- let arr_a = create_array(2u64.pow(12) as usize, true);
- let arr_b = create_array(2u64.pow(12) as usize, true);
-
- c.bench_function("sort nulls 2^12 limit 10", |b| {
- b.iter(|| bench_sort(&arr_a, &arr_b, Some(10)))
- });
- c.bench_function("sort nulls 2^12 limit 100", |b| {
- b.iter(|| bench_sort(&arr_a, &arr_b, Some(100)))
- });
- c.bench_function("sort nulls 2^12 limit 1000", |b| {
- b.iter(|| bench_sort(&arr_a, &arr_b, Some(1000)))
- });
- c.bench_function("sort nulls 2^12 limit 2^12", |b| {
- b.iter(|| bench_sort(&arr_a, &arr_b, Some(2u64.pow(12) as usize)))
- });
- }
-}
-
-criterion_group!(benches, add_benchmark);
-criterion_main!(benches);
diff --git a/rust/arrow/benches/take_kernels.rs b/rust/arrow/benches/take_kernels.rs
deleted file mode 100644
index 2853eb5..0000000
--- a/rust/arrow/benches/take_kernels.rs
+++ /dev/null
@@ -1,128 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#[macro_use]
-extern crate criterion;
-use criterion::Criterion;
-
-use rand::Rng;
-
-extern crate arrow;
-
-use arrow::compute::take;
-use arrow::datatypes::*;
-use arrow::util::test_util::seedable_rng;
-use arrow::{array::*, util::bench_util::*};
-
-fn create_random_index(size: usize, null_density: f32) -> UInt32Array {
- let mut rng = seedable_rng();
- let mut builder = UInt32Builder::new(size);
- for _ in 0..size {
- if rng.gen::<f32>() < null_density {
- builder.append_null().unwrap()
- } else {
- let value = rng.gen_range::<u32, _, _>(0u32, size as u32);
- builder.append_value(value).unwrap();
- }
- }
- builder.finish()
-}
-
-fn bench_take(values: &dyn Array, indices: &UInt32Array) {
- criterion::black_box(take(values, &indices, None).unwrap());
-}
-
-fn add_benchmark(c: &mut Criterion) {
- let values = create_primitive_array::<Int32Type>(512, 0.0);
- let indices = create_random_index(512, 0.0);
- c.bench_function("take i32 512", |b| b.iter(|| bench_take(&values, &indices)));
- let values = create_primitive_array::<Int32Type>(1024, 0.0);
- let indices = create_random_index(1024, 0.0);
- c.bench_function("take i32 1024", |b| {
- b.iter(|| bench_take(&values, &indices))
- });
-
- let indices = create_random_index(512, 0.5);
- c.bench_function("take i32 nulls 512", |b| {
- b.iter(|| bench_take(&values, &indices))
- });
- let values = create_primitive_array::<Int32Type>(1024, 0.0);
- let indices = create_random_index(1024, 0.5);
- c.bench_function("take i32 nulls 1024", |b| {
- b.iter(|| bench_take(&values, &indices))
- });
-
- let values = create_boolean_array(512, 0.0, 0.5);
- let indices = create_random_index(512, 0.0);
- c.bench_function("take bool 512", |b| {
- b.iter(|| bench_take(&values, &indices))
- });
- let values = create_boolean_array(1024, 0.0, 0.5);
- let indices = create_random_index(1024, 0.0);
- c.bench_function("take bool 1024", |b| {
- b.iter(|| bench_take(&values, &indices))
- });
-
- let values = create_boolean_array(512, 0.0, 0.5);
- let indices = create_random_index(512, 0.5);
- c.bench_function("take bool nulls 512", |b| {
- b.iter(|| bench_take(&values, &indices))
- });
- let values = create_boolean_array(1024, 0.0, 0.5);
- let indices = create_random_index(1024, 0.5);
- c.bench_function("take bool nulls 1024", |b| {
- b.iter(|| bench_take(&values, &indices))
- });
-
- let values = create_string_array::<i32>(512, 0.0);
- let indices = create_random_index(512, 0.0);
- c.bench_function("take str 512", |b| b.iter(|| bench_take(&values, &indices)));
-
- let values = create_string_array::<i32>(1024, 0.0);
- let indices = create_random_index(1024, 0.0);
- c.bench_function("take str 1024", |b| {
- b.iter(|| bench_take(&values, &indices))
- });
-
- let values = create_string_array::<i32>(512, 0.0);
- let indices = create_random_index(512, 0.5);
- c.bench_function("take str null indices 512", |b| {
- b.iter(|| bench_take(&values, &indices))
- });
-
- let values = create_string_array::<i32>(1024, 0.0);
- let indices = create_random_index(1024, 0.5);
- c.bench_function("take str null indices 1024", |b| {
- b.iter(|| bench_take(&values, &indices))
- });
-
- let values = create_string_array::<i32>(1024, 0.5);
-
- let indices = create_random_index(1024, 0.0);
- c.bench_function("take str null values 1024", |b| {
- b.iter(|| bench_take(&values, &indices))
- });
-
- let values = create_string_array::<i32>(1024, 0.5);
- let indices = create_random_index(1024, 0.5);
- c.bench_function("take str null values null indices 1024", |b| {
- b.iter(|| bench_take(&values, &indices))
- });
-}
-
-criterion_group!(benches, add_benchmark);
-criterion_main!(benches);
diff --git a/rust/arrow/build.rs b/rust/arrow/build.rs
deleted file mode 100644
index 2e3a711..0000000
--- a/rust/arrow/build.rs
+++ /dev/null
@@ -1,26 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use cfg_aliases::cfg_aliases;
-
-fn main() {
- println!("cargo:rerun-if-changed=build.rs");
- // Setup cfg aliases
- cfg_aliases! {
- simd: { all(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"), feature = "simd") },
- }
-}
diff --git a/rust/arrow/examples/builders.rs b/rust/arrow/examples/builders.rs
deleted file mode 100644
index 61cce0e..0000000
--- a/rust/arrow/examples/builders.rs
+++ /dev/null
@@ -1,131 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-///! Many builders are available to easily create different types of arrow arrays
-extern crate arrow;
-
-use std::sync::Arc;
-
-use arrow::array::{
- Array, ArrayData, BooleanArray, Int32Array, Int32Builder, ListArray, PrimitiveArray,
- StringArray, StructArray,
-};
-use arrow::buffer::Buffer;
-use arrow::datatypes::{DataType, Date64Type, Field, Time64NanosecondType, ToByteSlice};
-
-fn main() {
- // Primitive Arrays
- //
- // Primitive arrays are arrays of fixed-width primitive types (bool, u8, u16, u32,
- // u64, i8, i16, i32, i64, f32, f64)
-
- // Create a new builder with a capacity of 100
- let mut primitive_array_builder = Int32Builder::new(100);
-
- // Append an individual primitive value
- primitive_array_builder.append_value(55).unwrap();
-
- // Append a null value
- primitive_array_builder.append_null().unwrap();
-
- // Append a slice of primitive values
- primitive_array_builder.append_slice(&[39, 89, 12]).unwrap();
-
- // Append lots of values
- primitive_array_builder.append_null().unwrap();
- primitive_array_builder
- .append_slice(&(25..50).collect::<Vec<i32>>())
- .unwrap();
-
- // Build the `PrimitiveArray`
- let primitive_array = primitive_array_builder.finish();
- // Long arrays will have an ellipsis printed in the middle
- println!("{:?}", primitive_array);
-
- // Arrays can also be built from `Vec<Option<T>>`. `None`
- // represents a null value in the array.
- let date_array: PrimitiveArray<Date64Type> =
- vec![Some(1550902545147), None, Some(1550902545147)].into();
- println!("{:?}", date_array);
-
- let time_array: PrimitiveArray<Time64NanosecondType> =
- (0..100).collect::<Vec<i64>>().into();
- println!("{:?}", time_array);
-
- // We can build arrays directly from the underlying buffers.
-
- // BinaryArrays are arrays of byte arrays, where each byte array
- // is a slice of an underlying buffer.
-
- // Array data: ["hello", null, "parquet"]
- let values: [u8; 12] = [
- b'h', b'e', b'l', b'l', b'o', b'p', b'a', b'r', b'q', b'u', b'e', b't',
- ];
- let offsets: [i32; 4] = [0, 5, 5, 12];
-
- let array_data = ArrayData::builder(DataType::Utf8)
- .len(3)
- .add_buffer(Buffer::from(offsets.to_byte_slice()))
- .add_buffer(Buffer::from(&values[..]))
- .null_bit_buffer(Buffer::from([0b00000101]))
- .build();
- let binary_array = StringArray::from(array_data);
- println!("{:?}", binary_array);
-
- // ListArrays are similar to ByteArrays: they are arrays of other
- // arrays, where each child array is a slice of the underlying
- // buffer.
- let value_data = ArrayData::builder(DataType::Int32)
- .len(8)
- .add_buffer(Buffer::from(&[0, 1, 2, 3, 4, 5, 6, 7].to_byte_slice()))
- .build();
-
- // Construct a buffer for value offsets, for the nested array:
- // [[0, 1, 2], [3, 4, 5], [6, 7]]
- let value_offsets = Buffer::from(&[0, 3, 6, 8].to_byte_slice());
-
- // Construct a list array from the above two
- let list_data_type =
- DataType::List(Box::new(Field::new("item", DataType::Int32, false)));
- let list_data = ArrayData::builder(list_data_type)
- .len(3)
- .add_buffer(value_offsets)
- .add_child_data(value_data)
- .build();
- let list_array = ListArray::from(list_data);
-
- println!("{:?}", list_array);
-
- // StructArrays are arrays of tuples, where each tuple element is
- // from a child array. (In other words, they're like zipping
- // multiple columns into one and giving each subcolumn a label.)
-
- // StructArrays can be constructed using the StructArray::from
- // helper, which takes the underlying arrays and field types.
- let struct_array = StructArray::from(vec![
- (
- Field::new("b", DataType::Boolean, false),
- Arc::new(BooleanArray::from(vec![false, false, true, true]))
- as Arc<dyn Array>,
- ),
- (
- Field::new("c", DataType::Int32, false),
- Arc::new(Int32Array::from(vec![42, 28, 19, 31])),
- ),
- ]);
- println!("{:?}", struct_array);
-}
diff --git a/rust/arrow/examples/dynamic_types.rs b/rust/arrow/examples/dynamic_types.rs
deleted file mode 100644
index 58e4156..0000000
--- a/rust/arrow/examples/dynamic_types.rs
+++ /dev/null
@@ -1,101 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-///! This example demonstrates dealing with mixed types dynamically at runtime
-use std::sync::Arc;
-
-extern crate arrow;
-
-use arrow::array::*;
-use arrow::datatypes::*;
-use arrow::error::Result;
-use arrow::record_batch::*;
-
-fn main() -> Result<()> {
- // define schema
- let schema = Schema::new(vec![
- Field::new("id", DataType::Int32, false),
- Field::new(
- "nested",
- DataType::Struct(vec![
- Field::new("a", DataType::Utf8, false),
- Field::new("b", DataType::Float64, false),
- Field::new("c", DataType::Float64, false),
- ]),
- false,
- ),
- ]);
-
- // create some data
- let id = Int32Array::from(vec![1, 2, 3, 4, 5]);
-
- let nested = StructArray::from(vec![
- (
- Field::new("a", DataType::Utf8, false),
- Arc::new(StringArray::from(vec!["a", "b", "c", "d", "e"])) as Arc<dyn Array>,
- ),
- (
- Field::new("b", DataType::Float64, false),
- Arc::new(Float64Array::from(vec![1.1, 2.2, 3.3, 4.4, 5.5])),
- ),
- (
- Field::new("c", DataType::Float64, false),
- Arc::new(Float64Array::from(vec![2.2, 3.3, 4.4, 5.5, 6.6])),
- ),
- ]);
-
- // build a record batch
- let batch =
- RecordBatch::try_new(Arc::new(schema), vec![Arc::new(id), Arc::new(nested)])?;
-
- process(&batch);
- Ok(())
-}
-
-/// Create a new batch by performing a projection of id, nested.c
-fn process(batch: &RecordBatch) {
- let id = batch.column(0);
- let nested = batch
- .column(1)
- .as_any()
- .downcast_ref::<StructArray>()
- .unwrap();
-
- let _nested_b = nested
- .column(1)
- .as_any()
- .downcast_ref::<Float64Array>()
- .unwrap();
- let nested_c: &Float64Array = nested
- .column(2)
- .as_any()
- .downcast_ref::<Float64Array>()
- .unwrap();
-
- let projected_schema = Schema::new(vec![
- Field::new("id", DataType::Int32, false),
- Field::new("sum", DataType::Float64, false),
- ]);
-
- let _ = RecordBatch::try_new(
- Arc::new(projected_schema),
- vec![
- id.clone(), // NOTE: this is cloning the Arc not the array data
- Arc::new(Float64Array::from(nested_c.data().clone())),
- ],
- );
-}
diff --git a/rust/arrow/examples/read_csv.rs b/rust/arrow/examples/read_csv.rs
deleted file mode 100644
index 9e2b9c3..0000000
--- a/rust/arrow/examples/read_csv.rs
+++ /dev/null
@@ -1,43 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-extern crate arrow;
-
-use std::fs::File;
-use std::sync::Arc;
-
-use arrow::csv;
-use arrow::datatypes::{DataType, Field, Schema};
-#[cfg(feature = "prettyprint")]
-use arrow::util::pretty::print_batches;
-
-fn main() {
- let schema = Schema::new(vec![
- Field::new("city", DataType::Utf8, false),
- Field::new("lat", DataType::Float64, false),
- Field::new("lng", DataType::Float64, false),
- ]);
-
- let file = File::open("test/data/uk_cities.csv").unwrap();
-
- let mut csv = csv::Reader::new(file, Arc::new(schema), false, None, 1024, None, None);
- let _batch = csv.next().unwrap().unwrap();
- #[cfg(feature = "prettyprint")]
- {
- print_batches(&[_batch]).unwrap();
- }
-}
diff --git a/rust/arrow/examples/read_csv_infer_schema.rs b/rust/arrow/examples/read_csv_infer_schema.rs
deleted file mode 100644
index 93253e7..0000000
--- a/rust/arrow/examples/read_csv_infer_schema.rs
+++ /dev/null
@@ -1,36 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-extern crate arrow;
-
-use arrow::csv;
-#[cfg(feature = "prettyprint")]
-use arrow::util::pretty::print_batches;
-use std::fs::File;
-
-fn main() {
- let file = File::open("test/data/uk_cities_with_headers.csv").unwrap();
- let builder = csv::ReaderBuilder::new()
- .has_header(true)
- .infer_schema(Some(100));
- let mut csv = builder.build(file).unwrap();
- let _batch = csv.next().unwrap().unwrap();
- #[cfg(feature = "prettyprint")]
- {
- print_batches(&[_batch]).unwrap();
- }
-}
diff --git a/rust/arrow/examples/tensor_builder.rs b/rust/arrow/examples/tensor_builder.rs
deleted file mode 100644
index 1ef5392..0000000
--- a/rust/arrow/examples/tensor_builder.rs
+++ /dev/null
@@ -1,67 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-///! Tensor builder example
-extern crate arrow;
-
-use arrow::array::*; //{Int32BufferBuilder, Float32BufferBuilder};
-use arrow::buffer::Buffer;
-use arrow::datatypes::ToByteSlice;
-use arrow::error::Result;
-use arrow::tensor::{Float32Tensor, Int32Tensor};
-
-fn main() -> Result<()> {
- // Building a tensor using the buffer builder for Int32
- // The buffer builder will pad the appended numbers
- // to match the required size for each buffer
- let mut builder = Int32BufferBuilder::new(16);
- for i in 0..16 {
- builder.append(i);
- }
- let buf = builder.finish();
-
- // When building a tensor the buffer and shape are required
- // The new function will estimate the expected stride for the
- // storage data
- let tensor = Int32Tensor::try_new(buf, Some(vec![2, 8]), None, None)?;
- println!("Int32 Tensor");
- println!("{:?}", tensor);
-
- // Creating a tensor using float type buffer builder
- let mut builder = Float32BufferBuilder::new(4);
- builder.append(1.0);
- builder.append(2.0);
- builder.append(3.0);
- builder.append(4.0);
- let buf = builder.finish();
-
- // When building the tensor the buffer and shape are necessary
- // The new function will estimate the expected stride for the
- // storage data
- let tensor = Float32Tensor::try_new(buf, Some(vec![2, 2]), None, None)?;
- println!("\nFloat32 Tensor");
- println!("{:?}", tensor);
-
- // In order to build a tensor from an array the function to_byte_slice add the
- // required padding to the elements in the array.
- let buf = Buffer::from(&[0, 1, 2, 3, 4, 5, 6, 7, 9, 10].to_byte_slice());
- let tensor = Int32Tensor::try_new(buf, Some(vec![2, 5]), None, None)?;
- println!("\nInt32 Tensor");
- println!("{:?}", tensor);
-
- Ok(())
-}
diff --git a/rust/arrow/format-0ed34c83.patch b/rust/arrow/format-0ed34c83.patch
deleted file mode 100644
index 5da0a0c..0000000
--- a/rust/arrow/format-0ed34c83.patch
+++ /dev/null
@@ -1,220 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-diff --git a/format/Message.fbs b/format/Message.fbs
-index 1a7e0dfff..f1c18d765 100644
---- a/format/Message.fbs
-+++ b/format/Message.fbs
-@@ -28,7 +28,7 @@ namespace org.apache.arrow.flatbuf;
- /// Metadata about a field at some level of a nested type tree (but not
- /// its children).
- ///
--/// For example, a List<Int16> with values [[1, 2, 3], null, [4], [5, 6], null]
-+/// For example, a List<Int16> with values `[[1, 2, 3], null, [4], [5, 6], null]`
- /// would have {length: 5, null_count: 2} for its List node, and {length: 6,
- /// null_count: 0} for its Int16 node, as separate FieldNode structs
- struct FieldNode {
-diff --git a/format/Schema.fbs b/format/Schema.fbs
-index 3b37e5d85..3b00dd478 100644
---- a/format/Schema.fbs
-+++ b/format/Schema.fbs
-@@ -110,10 +110,11 @@ table FixedSizeList {
- /// not enforced.
- ///
- /// Map
-+/// ```text
- /// - child[0] entries: Struct
- /// - child[0] key: K
- /// - child[1] value: V
--///
-+/// ```
- /// Neither the "entries" field nor the "key" field may be nullable.
- ///
- /// The metadata is structured so that Arrow systems without special handling
-@@ -129,7 +130,7 @@ enum UnionMode:short { Sparse, Dense }
- /// A union is a complex type with children in Field
- /// By default ids in the type vector refer to the offsets in the children
- /// optionally typeIds provides an indirection between the child offset and the type id
--/// for each child typeIds[offset] is the id used in the type vector
-+/// for each child `typeIds[offset]` is the id used in the type vector
- table Union {
- mode: UnionMode;
- typeIds: [ int ]; // optional, describes typeid of each child.
-diff --git a/format/SparseTensor.fbs b/format/SparseTensor.fbs
-index 3fe8a7582..a6fd2f9e7 100644
---- a/format/SparseTensor.fbs
-+++ b/format/SparseTensor.fbs
-@@ -37,21 +37,21 @@ namespace org.apache.arrow.flatbuf;
- ///
- /// For example, let X be a 2x3x4x5 tensor, and it has the following
- /// 6 non-zero values:
--///
-+/// ```text
- /// X[0, 1, 2, 0] := 1
- /// X[1, 1, 2, 3] := 2
- /// X[0, 2, 1, 0] := 3
- /// X[0, 1, 3, 0] := 4
- /// X[0, 1, 2, 1] := 5
- /// X[1, 2, 0, 4] := 6
--///
-+/// ```
- /// In COO format, the index matrix of X is the following 4x6 matrix:
--///
-+/// ```text
- /// [[0, 0, 0, 0, 1, 1],
- /// [1, 1, 1, 2, 1, 2],
- /// [2, 2, 3, 1, 2, 0],
- /// [0, 1, 0, 0, 3, 4]]
--///
-+/// ```
- /// When isCanonical is true, the indices is sorted in lexicographical order
- /// (row-major order), and it does not have duplicated entries. Otherwise,
- /// the indices may not be sorted, or may have duplicated entries.
-@@ -86,26 +86,27 @@ table SparseMatrixIndexCSX {
-
- /// indptrBuffer stores the location and size of indptr array that
- /// represents the range of the rows.
-- /// The i-th row spans from indptr[i] to indptr[i+1] in the data.
-+ /// The i-th row spans from `indptr[i]` to `indptr[i+1]` in the data.
- /// The length of this array is 1 + (the number of rows), and the type
- /// of index value is long.
- ///
- /// For example, let X be the following 6x4 matrix:
-- ///
-+ /// ```text
- /// X := [[0, 1, 2, 0],
- /// [0, 0, 3, 0],
- /// [0, 4, 0, 5],
- /// [0, 0, 0, 0],
- /// [6, 0, 7, 8],
- /// [0, 9, 0, 0]].
-- ///
-+ /// ```
- /// The array of non-zero values in X is:
-- ///
-+ /// ```text
- /// values(X) = [1, 2, 3, 4, 5, 6, 7, 8, 9].
-- ///
-+ /// ```
- /// And the indptr of X is:
-- ///
-+ /// ```text
- /// indptr(X) = [0, 2, 3, 5, 5, 8, 10].
-+ /// ```
- indptrBuffer: Buffer (required);
-
- /// The type of values in indicesBuffer
-@@ -116,9 +117,9 @@ table SparseMatrixIndexCSX {
- /// The type of index value is long.
- ///
- /// For example, the indices of the above X is:
-- ///
-+ /// ```text
- /// indices(X) = [1, 2, 2, 1, 3, 0, 2, 3, 1].
-- ///
-+ /// ```
- /// Note that the indices are sorted in lexicographical order for each row.
- indicesBuffer: Buffer (required);
- }
-@@ -126,7 +127,7 @@ table SparseMatrixIndexCSX {
- /// Compressed Sparse Fiber (CSF) sparse tensor index.
- table SparseTensorIndexCSF {
- /// CSF is a generalization of compressed sparse row (CSR) index.
-- /// See [smith2017knl]: http://shaden.io/pub-files/smith2017knl.pdf
-+ /// See [smith2017knl](http://shaden.io/pub-files/smith2017knl.pdf)
- ///
- /// CSF index recursively compresses each dimension of a tensor into a set
- /// of prefix trees. Each path from a root to leaf forms one tensor
-@@ -135,7 +136,7 @@ table SparseTensorIndexCSF {
- ///
- /// For example, let X be a 2x3x4x5 tensor and let it have the following
- /// 8 non-zero values:
-- ///
-+ /// ```text
- /// X[0, 0, 0, 1] := 1
- /// X[0, 0, 0, 2] := 2
- /// X[0, 1, 0, 0] := 3
-@@ -144,9 +145,9 @@ table SparseTensorIndexCSF {
- /// X[1, 1, 1, 0] := 6
- /// X[1, 1, 1, 1] := 7
- /// X[1, 1, 1, 2] := 8
-- ///
-+ /// ```
- /// As a prefix tree this would be represented as:
-- ///
-+ /// ```text
- /// 0 1
- /// / \ |
- /// 0 1 1
-@@ -154,24 +155,24 @@ table SparseTensorIndexCSF {
- /// 0 0 1 1
- /// /| /| | /| |
- /// 1 2 0 2 0 0 1 2
--
-+ /// ```
- /// The type of values in indptrBuffers
- indptrType: Int (required);
-
- /// indptrBuffers stores the sparsity structure.
- /// Each two consecutive dimensions in a tensor correspond to a buffer in
-- /// indptrBuffers. A pair of consecutive values at indptrBuffers[dim][i]
-- /// and indptrBuffers[dim][i + 1] signify a range of nodes in
-- /// indicesBuffers[dim + 1] who are children of indicesBuffers[dim][i] node.
-+ /// indptrBuffers. A pair of consecutive values at `indptrBuffers[dim][i]`
-+ /// and `indptrBuffers[dim][i + 1]` signify a range of nodes in
-+ /// `indicesBuffers[dim + 1]` who are children of `indicesBuffers[dim][i]` node.
- ///
- /// For example, the indptrBuffers for the above X is:
-- ///
-+ /// ```text
- /// indptrBuffer(X) = [
- /// [0, 2, 3],
- /// [0, 1, 3, 4],
- /// [0, 2, 4, 5, 8]
- /// ].
-- ///
-+ /// ```
- indptrBuffers: [Buffer] (required);
-
- /// The type of values in indicesBuffers
-@@ -180,22 +181,22 @@ table SparseTensorIndexCSF {
- /// indicesBuffers stores values of nodes.
- /// Each tensor dimension corresponds to a buffer in indicesBuffers.
- /// For example, the indicesBuffers for the above X is:
-- ///
-+ /// ```text
- /// indicesBuffer(X) = [
- /// [0, 1],
- /// [0, 1, 1],
- /// [0, 0, 1, 1],
- /// [1, 2, 0, 2, 0, 0, 1, 2]
- /// ].
-- ///
-+ /// ```
- indicesBuffers: [Buffer] (required);
-
- /// axisOrder stores the sequence in which dimensions were traversed to
- /// produce the prefix tree.
- /// For example, the axisOrder for the above X is:
-- ///
-+ /// ```text
- /// axisOrder(X) = [0, 1, 2, 3].
-- ///
-+ /// ```
- axisOrder: [int] (required);
- }
-
diff --git a/rust/arrow/regen.sh b/rust/arrow/regen.sh
deleted file mode 100755
index 9d384b6b6..0000000
--- a/rust/arrow/regen.sh
+++ /dev/null
@@ -1,157 +0,0 @@
-#!/bin/bash -e
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-
-# Change to the toplevel Rust directory
-pushd $DIR/../../
-
-echo "Build flatc from source ..."
-
-FB_URL="https://github.com/google/flatbuffers"
-# https://github.com/google/flatbuffers/pull/6393
-FB_COMMIT="408cf5802415e1dea65fef7489a6c2f3740fb381"
-FB_DIR="rust/arrow/.flatbuffers"
-FLATC="$FB_DIR/bazel-bin/flatc"
-
-if [ -z $(which bazel) ]; then
- echo "bazel is required to build flatc"
- exit 1
-fi
-
-echo "Bazel version: $(bazel version | head -1 | awk -F':' '{print $2}')"
-
-if [ ! -e $FB_DIR ]; then
- echo "git clone $FB_URL ..."
- git clone -b master --no-tag --depth 1 $FB_URL $FB_DIR
-else
- echo "git pull $FB_URL ..."
- git -C $FB_DIR pull
-fi
-
-echo "hard reset to $FB_COMMIT"
-git -C $FB_DIR reset --hard $FB_COMMIT
-
-pushd $FB_DIR
-echo "run: bazel build :flatc ..."
-bazel build :flatc
-popd
-
-FB_PATCH="rust/arrow/format-0ed34c83.patch"
-echo "Patch flatbuffer files with ${FB_PATCH} for cargo doc"
-echo "NOTE: the patch MAY need update in case of changes in format/*.fbs"
-git apply --check ${FB_PATCH} && git apply ${FB_PATCH}
-
-# Execute the code generation:
-$FLATC --filename-suffix "" --rust -o rust/arrow/src/ipc/gen/ format/*.fbs
-
-# Reset changes to format/
-git checkout -- format
-
-# Now the files are wrongly named so we have to change that.
-popd
-pushd $DIR/src/ipc/gen
-
-PREFIX=$(cat <<'HEREDOC'
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#![allow(dead_code)]
-#![allow(unused_imports)]
-
-use std::{cmp::Ordering, mem};
-use flatbuffers::EndianScalar;
-
-HEREDOC
-)
-
-SCHEMA_IMPORT="\nuse crate::ipc::gen::Schema::*;"
-SPARSE_TENSOR_IMPORT="\nuse crate::ipc::gen::SparseTensor::*;"
-TENSOR_IMPORT="\nuse crate::ipc::gen::Tensor::*;"
-
-# For flatbuffer(1.12.0+), remove: use crate::${name}::\*;
-names=("File" "Message" "Schema" "SparseTensor" "Tensor")
-
-# Remove all generated lines we don't need
-for f in `ls *.rs`; do
- if [[ $f == "mod.rs" ]]; then
- continue
- fi
-
- echo "Modifying: $f"
- sed -i '' '/extern crate flatbuffers;/d' $f
- sed -i '' '/use self::flatbuffers::EndianScalar;/d' $f
- sed -i '' '/\#\[allow(unused_imports, dead_code)\]/d' $f
- sed -i '' '/pub mod org {/d' $f
- sed -i '' '/pub mod apache {/d' $f
- sed -i '' '/pub mod arrow {/d' $f
- sed -i '' '/pub mod flatbuf {/d' $f
- sed -i '' '/} \/\/ pub mod flatbuf/d' $f
- sed -i '' '/} \/\/ pub mod arrow/d' $f
- sed -i '' '/} \/\/ pub mod apache/d' $f
- sed -i '' '/} \/\/ pub mod org/d' $f
- sed -i '' '/use std::mem;/d' $f
- sed -i '' '/use std::cmp::Ordering;/d' $f
-
- # required by flatc 1.12.0+
- sed -i '' "/\#\!\[allow(unused_imports, dead_code)\]/d" $f
- for name in ${names[@]}; do
- sed -i '' "/use crate::${name}::\*;/d" $f
- sed -i '' "s/use self::flatbuffers::Verifiable;/use flatbuffers::Verifiable;/g" $f
- done
-
- # Replace all occurrences of "type__" with "type_", "TYPE__" with "TYPE_".
- sed -i '' 's/type__/type_/g' $f
- sed -i '' 's/TYPE__/TYPE_/g' $f
-
- # Some files need prefixes
- if [[ $f == "File.rs" ]]; then
- # Now prefix the file with the static contents
- echo -e "${PREFIX}" "${SCHEMA_IMPORT}" | cat - $f > temp && mv temp $f
- elif [[ $f == "Message.rs" ]]; then
- echo -e "${PREFIX}" "${SCHEMA_IMPORT}" "${SPARSE_TENSOR_IMPORT}" "${TENSOR_IMPORT}" | cat - $f > temp && mv temp $f
- elif [[ $f == "SparseTensor.rs" ]]; then
- echo -e "${PREFIX}" "${SCHEMA_IMPORT}" "${TENSOR_IMPORT}" | cat - $f > temp && mv temp $f
- elif [[ $f == "Tensor.rs" ]]; then
- echo -e "${PREFIX}" "${SCHEMA_IMPORT}" | cat - $f > temp && mv temp $f
- else
- echo "${PREFIX}" | cat - $f > temp && mv temp $f
- fi
-done
-
-# Return back to base directory
-popd
-cargo +stable fmt -- src/ipc/gen/*
-
-echo "DONE!"
-echo "Please run 'cargo doc' and 'cargo test' with nightly and stable, "
-echo "and fix possible errors or warnings!"
diff --git a/rust/arrow/src/alloc/alignment.rs b/rust/arrow/src/alloc/alignment.rs
deleted file mode 100644
index dbf4602..0000000
--- a/rust/arrow/src/alloc/alignment.rs
+++ /dev/null
@@ -1,119 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// NOTE: Below code is written for spatial/temporal prefetcher optimizations. Memory allocation
-// should align well with usage pattern of cache access and block sizes on layers of storage levels from
-// registers to non-volatile memory. These alignments are all cache aware alignments incorporated
-// from [cuneiform](https://crates.io/crates/cuneiform) crate. This approach mimicks Intel TBB's
-// cache_aligned_allocator which exploits cache locality and minimizes prefetch signals
-// resulting in less round trip time between the layers of storage.
-// For further info: https://software.intel.com/en-us/node/506094
-
-// 32-bit architecture and things other than netburst microarchitecture are using 64 bytes.
-/// Cache and allocation multiple alignment size
-#[cfg(target_arch = "x86")]
-pub const ALIGNMENT: usize = 1 << 6;
-
-// Intel x86_64:
-// L2D streamer from L1:
-// Loads data or instructions from memory to the second-level cache. To use the streamer,
-// organize the data or instructions in blocks of 128 bytes, aligned on 128 bytes.
-// - https://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-optimization-manual.pdf
-/// Cache and allocation multiple alignment size
-#[cfg(target_arch = "x86_64")]
-pub const ALIGNMENT: usize = 1 << 7;
-
-// 24Kc:
-// Data Line Size
-// - https://s3-eu-west-1.amazonaws.com/downloads-mips/documents/MD00346-2B-24K-DTS-04.00.pdf
-// - https://gitlab.e.foundation/e/devices/samsung/n7100/stable_android_kernel_samsung_smdk4412/commit/2dbac10263b2f3c561de68b4c369bc679352ccee
-/// Cache and allocation multiple alignment size
-#[cfg(target_arch = "mips")]
-pub const ALIGNMENT: usize = 1 << 5;
-/// Cache and allocation multiple alignment size
-#[cfg(target_arch = "mips64")]
-pub const ALIGNMENT: usize = 1 << 5;
-
-// Defaults for powerpc
-/// Cache and allocation multiple alignment size
-#[cfg(target_arch = "powerpc")]
-pub const ALIGNMENT: usize = 1 << 5;
-
-// Defaults for the ppc 64
-/// Cache and allocation multiple alignment size
-#[cfg(target_arch = "powerpc64")]
-pub const ALIGNMENT: usize = 1 << 6;
-
-// e.g.: sifive
-// - https://github.com/torvalds/linux/blob/master/Documentation/devicetree/bindings/riscv/sifive-l2-cache.txt#L41
-// in general all of them are the same.
-/// Cache and allocation multiple alignment size
-#[cfg(target_arch = "riscv")]
-pub const ALIGNMENT: usize = 1 << 6;
-
-// This size is same across all hardware for this architecture.
-// - https://docs.huihoo.com/doxygen/linux/kernel/3.7/arch_2s390_2include_2asm_2cache_8h.html
-/// Cache and allocation multiple alignment size
-#[cfg(target_arch = "s390x")]
-pub const ALIGNMENT: usize = 1 << 8;
-
-// This size is same across all hardware for this architecture.
-// - https://docs.huihoo.com/doxygen/linux/kernel/3.7/arch_2sparc_2include_2asm_2cache_8h.html#a9400cc2ba37e33279bdbc510a6311fb4
-/// Cache and allocation multiple alignment size
-#[cfg(target_arch = "sparc")]
-pub const ALIGNMENT: usize = 1 << 5;
-/// Cache and allocation multiple alignment size
-#[cfg(target_arch = "sparc64")]
-pub const ALIGNMENT: usize = 1 << 6;
-
-// On ARM cache line sizes are fixed. both v6 and v7.
-// Need to add board specific or platform specific things later.
-/// Cache and allocation multiple alignment size
-#[cfg(target_arch = "thumbv6")]
-pub const ALIGNMENT: usize = 1 << 5;
-/// Cache and allocation multiple alignment size
-#[cfg(target_arch = "thumbv7")]
-pub const ALIGNMENT: usize = 1 << 5;
-
-// Operating Systems cache size determines this.
-// Currently no way to determine this without runtime inference.
-/// Cache and allocation multiple alignment size
-#[cfg(target_arch = "wasm32")]
-pub const ALIGNMENT: usize = 1 << 6;
-
-// Same as v6 and v7.
-// List goes like that:
-// Cortex A, M, R, ARM v7, v7-M, Krait and NeoverseN uses this size.
-/// Cache and allocation multiple alignment size
-#[cfg(target_arch = "arm")]
-pub const ALIGNMENT: usize = 1 << 5;
-
-// Combined from 4 sectors. Volta says 128.
-// Prevent chunk optimizations better to go to the default size.
-// If you have smaller data with less padded functionality then use 32 with force option.
-// - https://devtalk.nvidia.com/default/topic/803600/variable-cache-line-width-/
-/// Cache and allocation multiple alignment size
-#[cfg(target_arch = "nvptx")]
-pub const ALIGNMENT: usize = 1 << 7;
-/// Cache and allocation multiple alignment size
-#[cfg(target_arch = "nvptx64")]
-pub const ALIGNMENT: usize = 1 << 7;
-
-// This size is same across all hardware for this architecture.
-/// Cache and allocation multiple alignment size
-#[cfg(target_arch = "aarch64")]
-pub const ALIGNMENT: usize = 1 << 6;
diff --git a/rust/arrow/src/alloc/mod.rs b/rust/arrow/src/alloc/mod.rs
deleted file mode 100644
index a225d32..0000000
--- a/rust/arrow/src/alloc/mod.rs
+++ /dev/null
@@ -1,136 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Defines memory-related functions, such as allocate/deallocate/reallocate memory
-//! regions, cache and allocation alignments.
-
-use std::mem::size_of;
-use std::ptr::NonNull;
-use std::{
- alloc::{handle_alloc_error, Layout},
- sync::atomic::AtomicIsize,
-};
-
-mod alignment;
-mod types;
-
-pub use alignment::ALIGNMENT;
-pub use types::NativeType;
-
-// If this number is not zero after all objects have been `drop`, there is a memory leak
-pub static mut ALLOCATIONS: AtomicIsize = AtomicIsize::new(0);
-
-#[inline]
-unsafe fn null_pointer<T: NativeType>() -> NonNull<T> {
- NonNull::new_unchecked(ALIGNMENT as *mut T)
-}
-
-/// Allocates a cache-aligned memory region of `size` bytes with uninitialized values.
-/// This is more performant than using [allocate_aligned_zeroed] when all bytes will have
-/// an unknown or non-zero value and is semantically similar to `malloc`.
-pub fn allocate_aligned<T: NativeType>(size: usize) -> NonNull<T> {
- unsafe {
- if size == 0 {
- null_pointer()
- } else {
- let size = size * size_of::<T>();
- ALLOCATIONS.fetch_add(size as isize, std::sync::atomic::Ordering::SeqCst);
-
- let layout = Layout::from_size_align_unchecked(size, ALIGNMENT);
- let raw_ptr = std::alloc::alloc(layout) as *mut T;
- NonNull::new(raw_ptr).unwrap_or_else(|| handle_alloc_error(layout))
- }
- }
-}
-
-/// Allocates a cache-aligned memory region of `size` bytes with `0` on all of them.
-/// This is more performant than using [allocate_aligned] and setting all bytes to zero
-/// and is semantically similar to `calloc`.
-pub fn allocate_aligned_zeroed<T: NativeType>(size: usize) -> NonNull<T> {
- unsafe {
- if size == 0 {
- null_pointer()
- } else {
- let size = size * size_of::<T>();
- ALLOCATIONS.fetch_add(size as isize, std::sync::atomic::Ordering::SeqCst);
-
- let layout = Layout::from_size_align_unchecked(size, ALIGNMENT);
- let raw_ptr = std::alloc::alloc_zeroed(layout) as *mut T;
- NonNull::new(raw_ptr).unwrap_or_else(|| handle_alloc_error(layout))
- }
- }
-}
-
-/// # Safety
-///
-/// This function is unsafe because undefined behavior can result if the caller does not ensure all
-/// of the following:
-///
-/// * ptr must denote a block of memory currently allocated via this allocator,
-///
-/// * size must be the same size that was used to allocate that block of memory,
-pub unsafe fn free_aligned<T: NativeType>(ptr: NonNull<T>, size: usize) {
- if ptr != null_pointer() {
- let size = size * size_of::<T>();
- ALLOCATIONS.fetch_sub(size as isize, std::sync::atomic::Ordering::SeqCst);
- std::alloc::dealloc(
- ptr.as_ptr() as *mut u8,
- Layout::from_size_align_unchecked(size, ALIGNMENT),
- );
- }
-}
-
-/// # Safety
-///
-/// This function is unsafe because undefined behavior can result if the caller does not ensure all
-/// of the following:
-///
-/// * ptr must be currently allocated via this allocator,
-///
-/// * new_size must be greater than zero.
-///
-/// * new_size, when rounded up to the nearest multiple of [ALIGNMENT], must not overflow (i.e.,
-/// the rounded value must be less than usize::MAX).
-pub unsafe fn reallocate<T: NativeType>(
- ptr: NonNull<T>,
- old_size: usize,
- new_size: usize,
-) -> NonNull<T> {
- let old_size = old_size * size_of::<T>();
- let new_size = new_size * size_of::<T>();
- if ptr == null_pointer() {
- return allocate_aligned(new_size);
- }
-
- if new_size == 0 {
- free_aligned(ptr, old_size);
- return null_pointer();
- }
-
- ALLOCATIONS.fetch_add(
- new_size as isize - old_size as isize,
- std::sync::atomic::Ordering::SeqCst,
- );
- let raw_ptr = std::alloc::realloc(
- ptr.as_ptr() as *mut u8,
- Layout::from_size_align_unchecked(old_size, ALIGNMENT),
- new_size,
- ) as *mut T;
- NonNull::new(raw_ptr).unwrap_or_else(|| {
- handle_alloc_error(Layout::from_size_align_unchecked(new_size, ALIGNMENT))
- })
-}
diff --git a/rust/arrow/src/alloc/types.rs b/rust/arrow/src/alloc/types.rs
deleted file mode 100644
index c1f0ef9..0000000
--- a/rust/arrow/src/alloc/types.rs
+++ /dev/null
@@ -1,71 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use crate::datatypes::DataType;
-
-/// A type that Rust's custom allocator knows how to allocate and deallocate.
-/// This is implemented for all Arrow's physical types whose in-memory representation
-/// matches Rust's physical types. Consider this trait sealed.
-/// # Safety
-/// Do not implement this trait.
-pub unsafe trait NativeType:
- Sized + Copy + std::fmt::Debug + std::fmt::Display + PartialEq + Default + Sized + 'static
-{
- type Bytes: AsRef<[u8]>;
-
- /// Whether a DataType is a valid type for this physical representation.
- fn is_valid(data_type: &DataType) -> bool;
-
- /// How this type represents itself as bytes in little endianess.
- /// This is used for IPC, where data is communicated with a specific endianess.
- fn to_le_bytes(&self) -> Self::Bytes;
-}
-
-macro_rules! create_native {
- ($native_ty:ty,$($impl_pattern:pat)|+) => {
- unsafe impl NativeType for $native_ty {
- type Bytes = [u8; std::mem::size_of::<Self>()];
-
- #[inline]
- fn to_le_bytes(&self) -> Self::Bytes {
- Self::to_le_bytes(*self)
- }
-
- #[inline]
- fn is_valid(data_type: &DataType) -> bool {
- matches!(data_type, $($impl_pattern)|+)
- }
- }
- };
-}
-
-create_native!(u8, DataType::UInt8);
-create_native!(u16, DataType::UInt16);
-create_native!(u32, DataType::UInt32);
-create_native!(u64, DataType::UInt64);
-create_native!(i8, DataType::Int8);
-create_native!(i16, DataType::Int16);
-create_native!(
- i32,
- DataType::Int32 | DataType::Date32 | DataType::Time32(_)
-);
-create_native!(
- i64,
- DataType::Int64 | DataType::Date64 | DataType::Time64(_) | DataType::Timestamp(_, _)
-);
-create_native!(f32, DataType::Float32);
-create_native!(f64, DataType::Float64);
diff --git a/rust/arrow/src/arch/avx512.rs b/rust/arrow/src/arch/avx512.rs
deleted file mode 100644
index 264532f..0000000
--- a/rust/arrow/src/arch/avx512.rs
+++ /dev/null
@@ -1,73 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-pub(crate) const AVX512_U8X64_LANES: usize = 64;
-
-#[target_feature(enable = "avx512f")]
-pub(crate) unsafe fn avx512_bin_and(left: &[u8], right: &[u8], res: &mut [u8]) {
- use core::arch::x86_64::{__m512i, _mm512_and_si512, _mm512_loadu_epi64};
-
- let l: __m512i = _mm512_loadu_epi64(left.as_ptr() as *const _);
- let r: __m512i = _mm512_loadu_epi64(right.as_ptr() as *const _);
- let f = _mm512_and_si512(l, r);
- let s = &f as *const __m512i as *const u8;
- let d = res.get_unchecked_mut(0) as *mut _ as *mut u8;
- std::ptr::copy_nonoverlapping(s, d, std::mem::size_of::<__m512i>());
-}
-
-#[target_feature(enable = "avx512f")]
-pub(crate) unsafe fn avx512_bin_or(left: &[u8], right: &[u8], res: &mut [u8]) {
- use core::arch::x86_64::{__m512i, _mm512_loadu_epi64, _mm512_or_si512};
-
- let l: __m512i = _mm512_loadu_epi64(left.as_ptr() as *const _);
- let r: __m512i = _mm512_loadu_epi64(right.as_ptr() as *const _);
- let f = _mm512_or_si512(l, r);
- let s = &f as *const __m512i as *const u8;
- let d = res.get_unchecked_mut(0) as *mut _ as *mut u8;
- std::ptr::copy_nonoverlapping(s, d, std::mem::size_of::<__m512i>());
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn test_bitwise_and_avx512() {
- let buf1 = [0b00110011u8; 64];
- let buf2 = [0b11110000u8; 64];
- let mut buf3 = [0b00000000; 64];
- unsafe {
- avx512_bin_and(&buf1, &buf2, &mut buf3);
- };
- for i in buf3.iter() {
- assert_eq!(&0b00110000u8, i);
- }
- }
-
- #[test]
- fn test_bitwise_or_avx512() {
- let buf1 = [0b00010011u8; 64];
- let buf2 = [0b11100000u8; 64];
- let mut buf3 = [0b00000000; 64];
- unsafe {
- avx512_bin_or(&buf1, &buf2, &mut buf3);
- };
- for i in buf3.iter() {
- assert_eq!(&0b11110011u8, i);
- }
- }
-}
diff --git a/rust/arrow/src/arch/mod.rs b/rust/arrow/src/arch/mod.rs
deleted file mode 100644
index 56d8f4c..0000000
--- a/rust/arrow/src/arch/mod.rs
+++ /dev/null
@@ -1,22 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-///
-/// Arch module contains architecture specific code.
-/// Be aware that not all machines have these specific operations available.
-#[cfg(all(target_arch = "x86_64", feature = "avx512"))]
-pub(crate) mod avx512;
diff --git a/rust/arrow/src/array/array.rs b/rust/arrow/src/array/array.rs
deleted file mode 100644
index 95a3117..0000000
--- a/rust/arrow/src/array/array.rs
+++ /dev/null
@@ -1,640 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::fmt;
-use std::sync::Arc;
-use std::{any::Any, convert::TryFrom};
-
-use super::*;
-use crate::array::equal_json::JsonEqual;
-use crate::buffer::{Buffer, MutableBuffer};
-use crate::error::Result;
-use crate::ffi;
-
-/// Trait for dealing with different types of array at runtime when the type of the
-/// array is not known in advance.
-pub trait Array: fmt::Debug + Send + Sync + JsonEqual {
- /// Returns the array as [`Any`](std::any::Any) so that it can be
- /// downcasted to a specific implementation.
- ///
- /// # Example:
- ///
- /// ```
- /// use std::sync::Arc;
- /// use arrow::array::Int32Array;
- /// use arrow::datatypes::{Schema, Field, DataType};
- /// use arrow::record_batch::RecordBatch;
- ///
- /// # fn main() -> arrow::error::Result<()> {
- /// let id = Int32Array::from(vec![1, 2, 3, 4, 5]);
- /// let batch = RecordBatch::try_new(
- /// Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])),
- /// vec![Arc::new(id)]
- /// )?;
- ///
- /// let int32array = batch
- /// .column(0)
- /// .as_any()
- /// .downcast_ref::<Int32Array>()
- /// .expect("Failed to downcast");
- /// # Ok(())
- /// # }
- /// ```
- fn as_any(&self) -> &Any;
-
- /// Returns a reference to the underlying data of this array.
- fn data(&self) -> &ArrayData;
-
- /// Returns a reference-counted pointer to the underlying data of this array.
- fn data_ref(&self) -> &ArrayData {
- self.data()
- }
-
- /// Returns a reference to the [`DataType`](crate::datatypes::DataType) of this array.
- ///
- /// # Example:
- ///
- /// ```
- /// use arrow::datatypes::DataType;
- /// use arrow::array::{Array, Int32Array};
- ///
- /// let array = Int32Array::from(vec![1, 2, 3, 4, 5]);
- ///
- /// assert_eq!(*array.data_type(), DataType::Int32);
- /// ```
- fn data_type(&self) -> &DataType {
- self.data_ref().data_type()
- }
-
- /// Returns a zero-copy slice of this array with the indicated offset and length.
- ///
- /// # Example:
- ///
- /// ```
- /// use arrow::array::{Array, Int32Array};
- ///
- /// let array = Int32Array::from(vec![1, 2, 3, 4, 5]);
- /// // Make slice over the values [2, 3, 4]
- /// let array_slice = array.slice(1, 3);
- ///
- /// assert_eq!(array_slice.as_ref(), &Int32Array::from(vec![2, 3, 4]));
- /// ```
- fn slice(&self, offset: usize, length: usize) -> ArrayRef {
- make_array(self.data_ref().slice(offset, length))
- }
-
- /// Returns the length (i.e., number of elements) of this array.
- ///
- /// # Example:
- ///
- /// ```
- /// use arrow::array::{Array, Int32Array};
- ///
- /// let array = Int32Array::from(vec![1, 2, 3, 4, 5]);
- ///
- /// assert_eq!(array.len(), 5);
- /// ```
- fn len(&self) -> usize {
- self.data_ref().len()
- }
-
- /// Returns whether this array is empty.
- ///
- /// # Example:
- ///
- /// ```
- /// use arrow::array::{Array, Int32Array};
- ///
- /// let array = Int32Array::from(vec![1, 2, 3, 4, 5]);
- ///
- /// assert_eq!(array.is_empty(), false);
- /// ```
- fn is_empty(&self) -> bool {
- self.data_ref().is_empty()
- }
-
- /// Returns the offset into the underlying data used by this array(-slice).
- /// Note that the underlying data can be shared by many arrays.
- /// This defaults to `0`.
- ///
- /// # Example:
- ///
- /// ```
- /// use arrow::array::{Array, Int32Array};
- ///
- /// let array = Int32Array::from(vec![1, 2, 3, 4, 5]);
- /// // Make slice over the values [2, 3, 4]
- /// let array_slice = array.slice(1, 3);
- ///
- /// assert_eq!(array.offset(), 0);
- /// assert_eq!(array_slice.offset(), 1);
- /// ```
- fn offset(&self) -> usize {
- self.data_ref().offset()
- }
-
- /// Returns whether the element at `index` is null.
- /// When using this function on a slice, the index is relative to the slice.
- ///
- /// # Example:
- ///
- /// ```
- /// use arrow::array::{Array, Int32Array};
- ///
- /// let array = Int32Array::from(vec![Some(1), None]);
- ///
- /// assert_eq!(array.is_null(0), false);
- /// assert_eq!(array.is_null(1), true);
- /// ```
- fn is_null(&self, index: usize) -> bool {
- self.data_ref().is_null(index)
- }
-
- /// Returns whether the element at `index` is not null.
- /// When using this function on a slice, the index is relative to the slice.
- ///
- /// # Example:
- ///
- /// ```
- /// use arrow::array::{Array, Int32Array};
- ///
- /// let array = Int32Array::from(vec![Some(1), None]);
- ///
- /// assert_eq!(array.is_valid(0), true);
- /// assert_eq!(array.is_valid(1), false);
- /// ```
- fn is_valid(&self, index: usize) -> bool {
- self.data_ref().is_valid(index)
- }
-
- /// Returns the total number of null values in this array.
- ///
- /// # Example:
- ///
- /// ```
- /// use arrow::array::{Array, Int32Array};
- ///
- /// // Construct an array with values [1, NULL, NULL]
- /// let array = Int32Array::from(vec![Some(1), None, None]);
- ///
- /// assert_eq!(array.null_count(), 2);
- /// ```
- fn null_count(&self) -> usize {
- self.data_ref().null_count()
- }
-
- /// Returns the total number of bytes of memory occupied by the buffers owned by this array.
- fn get_buffer_memory_size(&self) -> usize;
-
- /// Returns the total number of bytes of memory occupied physically by this array.
- fn get_array_memory_size(&self) -> usize;
-
- /// returns two pointers that represent this array in the C Data Interface (FFI)
- fn to_raw(
- &self,
- ) -> Result<(*const ffi::FFI_ArrowArray, *const ffi::FFI_ArrowSchema)> {
- let data = self.data().clone();
- let array = ffi::ArrowArray::try_from(data)?;
- Ok(ffi::ArrowArray::into_raw(array))
- }
-}
-
-/// A reference-counted reference to a generic `Array`.
-pub type ArrayRef = Arc<Array>;
-
-/// Constructs an array using the input `data`.
-/// Returns a reference-counted `Array` instance.
-pub fn make_array(data: ArrayData) -> ArrayRef {
- match data.data_type() {
- DataType::Boolean => Arc::new(BooleanArray::from(data)) as ArrayRef,
- DataType::Int8 => Arc::new(Int8Array::from(data)) as ArrayRef,
- DataType::Int16 => Arc::new(Int16Array::from(data)) as ArrayRef,
- DataType::Int32 => Arc::new(Int32Array::from(data)) as ArrayRef,
- DataType::Int64 => Arc::new(Int64Array::from(data)) as ArrayRef,
- DataType::UInt8 => Arc::new(UInt8Array::from(data)) as ArrayRef,
- DataType::UInt16 => Arc::new(UInt16Array::from(data)) as ArrayRef,
- DataType::UInt32 => Arc::new(UInt32Array::from(data)) as ArrayRef,
- DataType::UInt64 => Arc::new(UInt64Array::from(data)) as ArrayRef,
- DataType::Float16 => panic!("Float16 datatype not supported"),
- DataType::Float32 => Arc::new(Float32Array::from(data)) as ArrayRef,
- DataType::Float64 => Arc::new(Float64Array::from(data)) as ArrayRef,
- DataType::Date32 => Arc::new(Date32Array::from(data)) as ArrayRef,
- DataType::Date64 => Arc::new(Date64Array::from(data)) as ArrayRef,
- DataType::Time32(TimeUnit::Second) => {
- Arc::new(Time32SecondArray::from(data)) as ArrayRef
- }
- DataType::Time32(TimeUnit::Millisecond) => {
- Arc::new(Time32MillisecondArray::from(data)) as ArrayRef
- }
- DataType::Time64(TimeUnit::Microsecond) => {
- Arc::new(Time64MicrosecondArray::from(data)) as ArrayRef
- }
- DataType::Time64(TimeUnit::Nanosecond) => {
- Arc::new(Time64NanosecondArray::from(data)) as ArrayRef
- }
- DataType::Timestamp(TimeUnit::Second, _) => {
- Arc::new(TimestampSecondArray::from(data)) as ArrayRef
- }
- DataType::Timestamp(TimeUnit::Millisecond, _) => {
- Arc::new(TimestampMillisecondArray::from(data)) as ArrayRef
- }
- DataType::Timestamp(TimeUnit::Microsecond, _) => {
- Arc::new(TimestampMicrosecondArray::from(data)) as ArrayRef
- }
- DataType::Timestamp(TimeUnit::Nanosecond, _) => {
- Arc::new(TimestampNanosecondArray::from(data)) as ArrayRef
- }
- DataType::Interval(IntervalUnit::YearMonth) => {
- Arc::new(IntervalYearMonthArray::from(data)) as ArrayRef
- }
- DataType::Interval(IntervalUnit::DayTime) => {
- Arc::new(IntervalDayTimeArray::from(data)) as ArrayRef
- }
- DataType::Duration(TimeUnit::Second) => {
- Arc::new(DurationSecondArray::from(data)) as ArrayRef
- }
- DataType::Duration(TimeUnit::Millisecond) => {
- Arc::new(DurationMillisecondArray::from(data)) as ArrayRef
- }
- DataType::Duration(TimeUnit::Microsecond) => {
- Arc::new(DurationMicrosecondArray::from(data)) as ArrayRef
- }
- DataType::Duration(TimeUnit::Nanosecond) => {
- Arc::new(DurationNanosecondArray::from(data)) as ArrayRef
- }
- DataType::Binary => Arc::new(BinaryArray::from(data)) as ArrayRef,
- DataType::LargeBinary => Arc::new(LargeBinaryArray::from(data)) as ArrayRef,
- DataType::FixedSizeBinary(_) => {
- Arc::new(FixedSizeBinaryArray::from(data)) as ArrayRef
- }
- DataType::Utf8 => Arc::new(StringArray::from(data)) as ArrayRef,
- DataType::LargeUtf8 => Arc::new(LargeStringArray::from(data)) as ArrayRef,
- DataType::List(_) => Arc::new(ListArray::from(data)) as ArrayRef,
- DataType::LargeList(_) => Arc::new(LargeListArray::from(data)) as ArrayRef,
- DataType::Struct(_) => Arc::new(StructArray::from(data)) as ArrayRef,
- DataType::Union(_) => Arc::new(UnionArray::from(data)) as ArrayRef,
- DataType::FixedSizeList(_, _) => {
- Arc::new(FixedSizeListArray::from(data)) as ArrayRef
- }
- DataType::Dictionary(ref key_type, _) => match key_type.as_ref() {
- DataType::Int8 => {
- Arc::new(DictionaryArray::<Int8Type>::from(data)) as ArrayRef
- }
- DataType::Int16 => {
- Arc::new(DictionaryArray::<Int16Type>::from(data)) as ArrayRef
- }
- DataType::Int32 => {
- Arc::new(DictionaryArray::<Int32Type>::from(data)) as ArrayRef
- }
- DataType::Int64 => {
- Arc::new(DictionaryArray::<Int64Type>::from(data)) as ArrayRef
- }
- DataType::UInt8 => {
- Arc::new(DictionaryArray::<UInt8Type>::from(data)) as ArrayRef
- }
- DataType::UInt16 => {
- Arc::new(DictionaryArray::<UInt16Type>::from(data)) as ArrayRef
- }
- DataType::UInt32 => {
- Arc::new(DictionaryArray::<UInt32Type>::from(data)) as ArrayRef
- }
- DataType::UInt64 => {
- Arc::new(DictionaryArray::<UInt64Type>::from(data)) as ArrayRef
- }
- dt => panic!("Unexpected dictionary key type {:?}", dt),
- },
- DataType::Null => Arc::new(NullArray::from(data)) as ArrayRef,
- DataType::Decimal(_, _) => Arc::new(DecimalArray::from(data)) as ArrayRef,
- dt => panic!("Unexpected data type {:?}", dt),
- }
-}
-
-/// Creates a new empty array
-pub fn new_empty_array(data_type: &DataType) -> ArrayRef {
- let data = ArrayData::new_empty(data_type);
- make_array(data)
-}
-/// Creates a new array of `data_type` of length `length` filled entirely of `NULL` values
-pub fn new_null_array(data_type: &DataType, length: usize) -> ArrayRef {
- // context: https://github.com/apache/arrow/pull/9469#discussion_r574761687
- match data_type {
- DataType::Null => Arc::new(NullArray::new(length)),
- DataType::Boolean => {
- let null_buf: Buffer = MutableBuffer::new_null(length).into();
- make_array(ArrayData::new(
- data_type.clone(),
- length,
- Some(length),
- Some(null_buf.clone()),
- 0,
- vec![null_buf],
- vec![],
- ))
- }
- DataType::Int8 => new_null_sized_array::<Int8Type>(data_type, length),
- DataType::UInt8 => new_null_sized_array::<UInt8Type>(data_type, length),
- DataType::Int16 => new_null_sized_array::<Int16Type>(data_type, length),
- DataType::UInt16 => new_null_sized_array::<UInt16Type>(data_type, length),
- DataType::Float16 => unreachable!(),
- DataType::Int32 => new_null_sized_array::<Int32Type>(data_type, length),
- DataType::UInt32 => new_null_sized_array::<UInt32Type>(data_type, length),
- DataType::Float32 => new_null_sized_array::<Float32Type>(data_type, length),
- DataType::Date32 => new_null_sized_array::<Date32Type>(data_type, length),
- // expanding this into Date23{unit}Type results in needless branching
- DataType::Time32(_) => new_null_sized_array::<Int32Type>(data_type, length),
- DataType::Int64 => new_null_sized_array::<Int64Type>(data_type, length),
- DataType::UInt64 => new_null_sized_array::<UInt64Type>(data_type, length),
- DataType::Float64 => new_null_sized_array::<Float64Type>(data_type, length),
- DataType::Date64 => new_null_sized_array::<Date64Type>(data_type, length),
- // expanding this into Timestamp{unit}Type results in needless branching
- DataType::Timestamp(_, _) => new_null_sized_array::<Int64Type>(data_type, length),
- DataType::Time64(_) => new_null_sized_array::<Int64Type>(data_type, length),
- DataType::Duration(_) => new_null_sized_array::<Int64Type>(data_type, length),
- DataType::Interval(unit) => match unit {
- IntervalUnit::YearMonth => {
- new_null_sized_array::<IntervalYearMonthType>(data_type, length)
- }
- IntervalUnit::DayTime => {
- new_null_sized_array::<IntervalDayTimeType>(data_type, length)
- }
- },
- DataType::FixedSizeBinary(value_len) => make_array(ArrayData::new(
- data_type.clone(),
- length,
- Some(length),
- Some(MutableBuffer::new_null(length).into()),
- 0,
- vec![Buffer::from(vec![0u8; *value_len as usize * length])],
- vec![],
- )),
- DataType::Binary | DataType::Utf8 => {
- new_null_binary_array::<i32>(data_type, length)
- }
- DataType::LargeBinary | DataType::LargeUtf8 => {
- new_null_binary_array::<i64>(data_type, length)
- }
- DataType::List(field) => {
- new_null_list_array::<i32>(data_type, field.data_type(), length)
- }
- DataType::LargeList(field) => {
- new_null_list_array::<i64>(data_type, field.data_type(), length)
- }
- DataType::FixedSizeList(field, value_len) => make_array(ArrayData::new(
- data_type.clone(),
- length,
- Some(length),
- Some(MutableBuffer::new_null(length).into()),
- 0,
- vec![],
- vec![
- new_null_array(field.data_type(), *value_len as usize * length)
- .data()
- .clone(),
- ],
- )),
- DataType::Struct(fields) => make_array(ArrayData::new(
- data_type.clone(),
- length,
- Some(length),
- Some(MutableBuffer::new_null(length).into()),
- 0,
- vec![],
- fields
- .iter()
- .map(|field| ArrayData::new_empty(field.data_type()))
- .collect(),
- )),
- DataType::Union(_) => {
- unimplemented!("Creating null Union array not yet supported")
- }
- DataType::Dictionary(key, value) => {
- let keys = new_null_array(key, length);
- let keys = keys.data();
-
- make_array(ArrayData::new(
- data_type.clone(),
- length,
- Some(length),
- keys.null_buffer().cloned(),
- 0,
- keys.buffers().into(),
- vec![new_empty_array(value.as_ref()).data().clone()],
- ))
- }
- DataType::Decimal(_, _) => {
- unimplemented!("Creating null Decimal array not yet supported")
- }
- }
-}
-
-#[inline]
-fn new_null_list_array<OffsetSize: OffsetSizeTrait>(
- data_type: &DataType,
- child_data_type: &DataType,
- length: usize,
-) -> ArrayRef {
- make_array(ArrayData::new(
- data_type.clone(),
- length,
- Some(length),
- Some(MutableBuffer::new_null(length).into()),
- 0,
- vec![Buffer::from(
- vec![OffsetSize::zero(); length + 1].to_byte_slice(),
- )],
- vec![ArrayData::new_empty(child_data_type)],
- ))
-}
-
-#[inline]
-fn new_null_binary_array<OffsetSize: OffsetSizeTrait>(
- data_type: &DataType,
- length: usize,
-) -> ArrayRef {
- make_array(ArrayData::new(
- data_type.clone(),
- length,
- Some(length),
- Some(MutableBuffer::new_null(length).into()),
- 0,
- vec![
- Buffer::from(vec![OffsetSize::zero(); length + 1].to_byte_slice()),
- MutableBuffer::new(0).into(),
- ],
- vec![],
- ))
-}
-
-#[inline]
-fn new_null_sized_array<T: ArrowPrimitiveType>(
- data_type: &DataType,
- length: usize,
-) -> ArrayRef {
- make_array(ArrayData::new(
- data_type.clone(),
- length,
- Some(length),
- Some(MutableBuffer::new_null(length).into()),
- 0,
- vec![Buffer::from(vec![0u8; length * T::get_byte_width()])],
- vec![],
- ))
-}
-
-/// Creates a new array from two FFI pointers. Used to import arrays from the C Data Interface
-/// # Safety
-/// Assumes that these pointers represent valid C Data Interfaces, both in memory
-/// representation and lifetime via the `release` mechanism.
-pub unsafe fn make_array_from_raw(
- array: *const ffi::FFI_ArrowArray,
- schema: *const ffi::FFI_ArrowSchema,
-) -> Result<ArrayRef> {
- let array = ffi::ArrowArray::try_from_raw(array, schema)?;
- let data = ArrayData::try_from(array)?;
- Ok(make_array(data))
-}
-// Helper function for printing potentially long arrays.
-pub(super) fn print_long_array<A, F>(
- array: &A,
- f: &mut fmt::Formatter,
- print_item: F,
-) -> fmt::Result
-where
- A: Array,
- F: Fn(&A, usize, &mut fmt::Formatter) -> fmt::Result,
-{
- let head = std::cmp::min(10, array.len());
-
- for i in 0..head {
- if array.is_null(i) {
- writeln!(f, " null,")?;
- } else {
- write!(f, " ")?;
- print_item(&array, i, f)?;
- writeln!(f, ",")?;
- }
- }
- if array.len() > 10 {
- if array.len() > 20 {
- writeln!(f, " ...{} elements...,", array.len() - 20)?;
- }
-
- let tail = std::cmp::max(head, array.len() - 10);
-
- for i in tail..array.len() {
- if array.is_null(i) {
- writeln!(f, " null,")?;
- } else {
- write!(f, " ")?;
- print_item(&array, i, f)?;
- writeln!(f, ",")?;
- }
- }
- }
- Ok(())
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- #[test]
- fn test_empty_primitive() {
- let array = new_empty_array(&DataType::Int32);
- let a = array.as_any().downcast_ref::<Int32Array>().unwrap();
- assert_eq!(a.len(), 0);
- let expected: &[i32] = &[];
- assert_eq!(a.values(), expected);
- }
-
- #[test]
- fn test_empty_variable_sized() {
- let array = new_empty_array(&DataType::Utf8);
- let a = array.as_any().downcast_ref::<StringArray>().unwrap();
- assert_eq!(a.len(), 0);
- assert_eq!(a.value_offsets()[0], 0i32);
- }
-
- #[test]
- fn test_empty_list_primitive() {
- let data_type =
- DataType::List(Box::new(Field::new("item", DataType::Int32, false)));
- let array = new_empty_array(&data_type);
- let a = array.as_any().downcast_ref::<ListArray>().unwrap();
- assert_eq!(a.len(), 0);
- assert_eq!(a.value_offsets()[0], 0i32);
- }
-
- #[test]
- fn test_null_boolean() {
- let array = new_null_array(&DataType::Boolean, 9);
- let a = array.as_any().downcast_ref::<BooleanArray>().unwrap();
- assert_eq!(a.len(), 9);
- for i in 0..9 {
- assert!(a.is_null(i));
- }
- }
-
- #[test]
- fn test_null_primitive() {
- let array = new_null_array(&DataType::Int32, 9);
- let a = array.as_any().downcast_ref::<Int32Array>().unwrap();
- assert_eq!(a.len(), 9);
- for i in 0..9 {
- assert!(a.is_null(i));
- }
- }
-
- #[test]
- fn test_null_variable_sized() {
- let array = new_null_array(&DataType::Utf8, 9);
- let a = array.as_any().downcast_ref::<StringArray>().unwrap();
- assert_eq!(a.len(), 9);
- assert_eq!(a.value_offsets()[9], 0i32);
- for i in 0..9 {
- assert!(a.is_null(i));
- }
- }
-
- #[test]
- fn test_null_list_primitive() {
- let data_type =
- DataType::List(Box::new(Field::new("item", DataType::Int32, true)));
- let array = new_null_array(&data_type, 9);
- let a = array.as_any().downcast_ref::<ListArray>().unwrap();
- assert_eq!(a.len(), 9);
- assert_eq!(a.value_offsets()[9], 0i32);
- for i in 0..9 {
- assert!(a.is_null(i));
- }
- }
-
- #[test]
- fn test_null_dictionary() {
- let values = vec![None, None, None, None, None, None, None, None, None]
- as Vec<Option<&str>>;
-
- let array: DictionaryArray<Int8Type> = values.into_iter().collect();
- let array = Arc::new(array) as ArrayRef;
-
- let null_array = new_null_array(array.data_type(), 9);
- assert_eq!(&array, &null_array);
- assert_eq!(
- array.data().buffers()[0].len(),
- null_array.data().buffers()[0].len()
- );
- }
-}
diff --git a/rust/arrow/src/array/array_binary.rs b/rust/arrow/src/array/array_binary.rs
deleted file mode 100644
index bd04afa..0000000
--- a/rust/arrow/src/array/array_binary.rs
+++ /dev/null
@@ -1,1157 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::convert::{From, TryInto};
-use std::fmt;
-use std::mem;
-use std::{any::Any, iter::FromIterator};
-
-use super::{
- array::print_long_array, raw_pointer::RawPtrBox, Array, ArrayData,
- FixedSizeListArray, GenericBinaryIter, GenericListArray, OffsetSizeTrait,
-};
-use crate::buffer::Buffer;
-use crate::error::ArrowError;
-use crate::util::bit_util;
-use crate::{buffer::MutableBuffer, datatypes::DataType};
-
-/// Like OffsetSizeTrait, but specialized for Binary
-// This allow us to expose a constant datatype for the GenericBinaryArray
-pub trait BinaryOffsetSizeTrait: OffsetSizeTrait {
- const DATA_TYPE: DataType;
-}
-
-impl BinaryOffsetSizeTrait for i32 {
- const DATA_TYPE: DataType = DataType::Binary;
-}
-
-impl BinaryOffsetSizeTrait for i64 {
- const DATA_TYPE: DataType = DataType::LargeBinary;
-}
-
-pub struct GenericBinaryArray<OffsetSize: BinaryOffsetSizeTrait> {
- data: ArrayData,
- value_offsets: RawPtrBox<OffsetSize>,
- value_data: RawPtrBox<u8>,
-}
-
-impl<OffsetSize: BinaryOffsetSizeTrait> GenericBinaryArray<OffsetSize> {
- /// Returns the length for value at index `i`.
- #[inline]
- pub fn value_length(&self, i: usize) -> OffsetSize {
- let offsets = self.value_offsets();
- offsets[i + 1] - offsets[i]
- }
-
- /// Returns a clone of the value data buffer
- pub fn value_data(&self) -> Buffer {
- self.data.buffers()[1].clone()
- }
-
- /// Returns the offset values in the offsets buffer
- #[inline]
- pub fn value_offsets(&self) -> &[OffsetSize] {
- // Soundness
- // pointer alignment & location is ensured by RawPtrBox
- // buffer bounds/offset is ensured by the ArrayData instance.
- unsafe {
- std::slice::from_raw_parts(
- self.value_offsets.as_ptr().add(self.data.offset()),
- self.len() + 1,
- )
- }
- }
-
- /// Returns the element at index `i` as bytes slice
- /// # Safety
- /// Caller is responsible for ensuring that the index is within the bounds of the array
- pub unsafe fn value_unchecked(&self, i: usize) -> &[u8] {
- let end = *self.value_offsets().get_unchecked(i + 1);
- let start = *self.value_offsets().get_unchecked(i);
-
- // Soundness
- // pointer alignment & location is ensured by RawPtrBox
- // buffer bounds/offset is ensured by the value_offset invariants
-
- // Safety of `to_isize().unwrap()`
- // `start` and `end` are &OffsetSize, which is a generic type that implements the
- // OffsetSizeTrait. Currently, only i32 and i64 implement OffsetSizeTrait,
- // both of which should cleanly cast to isize on an architecture that supports
- // 32/64-bit offsets
- std::slice::from_raw_parts(
- self.value_data.as_ptr().offset(start.to_isize().unwrap()),
- (end - start).to_usize().unwrap(),
- )
- }
-
- /// Returns the element at index `i` as bytes slice
- pub fn value(&self, i: usize) -> &[u8] {
- assert!(i < self.data.len(), "BinaryArray out of bounds access");
- //Soundness: length checked above, offset buffer length is 1 larger than logical array length
- let end = unsafe { self.value_offsets().get_unchecked(i + 1) };
- let start = unsafe { self.value_offsets().get_unchecked(i) };
-
- // Soundness
- // pointer alignment & location is ensured by RawPtrBox
- // buffer bounds/offset is ensured by the value_offset invariants
-
- // Safety of `to_isize().unwrap()`
- // `start` and `end` are &OffsetSize, which is a generic type that implements the
- // OffsetSizeTrait. Currently, only i32 and i64 implement OffsetSizeTrait,
- // both of which should cleanly cast to isize on an architecture that supports
- // 32/64-bit offsets
- unsafe {
- std::slice::from_raw_parts(
- self.value_data.as_ptr().offset(start.to_isize().unwrap()),
- (*end - *start).to_usize().unwrap(),
- )
- }
- }
-
- /// Creates a [GenericBinaryArray] from a vector of byte slices
- pub fn from_vec(v: Vec<&[u8]>) -> Self {
- let mut offsets = Vec::with_capacity(v.len() + 1);
- let mut values = Vec::new();
- let mut length_so_far: OffsetSize = OffsetSize::zero();
- offsets.push(length_so_far);
- for s in &v {
- length_so_far += OffsetSize::from_usize(s.len()).unwrap();
- offsets.push(length_so_far);
- values.extend_from_slice(s);
- }
- let array_data = ArrayData::builder(OffsetSize::DATA_TYPE)
- .len(v.len())
- .add_buffer(Buffer::from_slice_ref(&offsets))
- .add_buffer(Buffer::from_slice_ref(&values))
- .build();
- GenericBinaryArray::<OffsetSize>::from(array_data)
- }
-
- /// Creates a [GenericBinaryArray] from a vector of Optional (null) byte slices
- pub fn from_opt_vec(v: Vec<Option<&[u8]>>) -> Self {
- v.into_iter().collect()
- }
-
- fn from_list(v: GenericListArray<OffsetSize>) -> Self {
- assert_eq!(
- v.data_ref().child_data()[0].child_data().len(),
- 0,
- "BinaryArray can only be created from list array of u8 values \
- (i.e. List<PrimitiveArray<u8>>)."
- );
- assert_eq!(
- v.data_ref().child_data()[0].data_type(),
- &DataType::UInt8,
- "BinaryArray can only be created from List<u8> arrays, mismatched data types."
- );
-
- let mut builder = ArrayData::builder(OffsetSize::DATA_TYPE)
- .len(v.len())
- .add_buffer(v.data_ref().buffers()[0].clone())
- .add_buffer(v.data_ref().child_data()[0].buffers()[0].clone());
- if let Some(bitmap) = v.data_ref().null_bitmap() {
- builder = builder.null_bit_buffer(bitmap.bits.clone())
- }
-
- let data = builder.build();
- Self::from(data)
- }
-}
-
-impl<'a, T: BinaryOffsetSizeTrait> GenericBinaryArray<T> {
- /// constructs a new iterator
- pub fn iter(&'a self) -> GenericBinaryIter<'a, T> {
- GenericBinaryIter::<'a, T>::new(&self)
- }
-}
-
-impl<OffsetSize: BinaryOffsetSizeTrait> fmt::Debug for GenericBinaryArray<OffsetSize> {
- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- let prefix = if OffsetSize::is_large() { "Large" } else { "" };
-
- write!(f, "{}BinaryArray\n[\n", prefix)?;
- print_long_array(self, f, |array, index, f| {
- fmt::Debug::fmt(&array.value(index), f)
- })?;
- write!(f, "]")
- }
-}
-
-impl<OffsetSize: BinaryOffsetSizeTrait> Array for GenericBinaryArray<OffsetSize> {
- fn as_any(&self) -> &Any {
- self
- }
-
- fn data(&self) -> &ArrayData {
- &self.data
- }
-
- /// Returns the total number of bytes of memory occupied by the buffers owned by this [$name].
- fn get_buffer_memory_size(&self) -> usize {
- self.data.get_buffer_memory_size()
- }
-
- /// Returns the total number of bytes of memory occupied physically by this [$name].
- fn get_array_memory_size(&self) -> usize {
- self.data.get_array_memory_size() + mem::size_of_val(self)
- }
-}
-
-impl<OffsetSize: BinaryOffsetSizeTrait> From<ArrayData>
- for GenericBinaryArray<OffsetSize>
-{
- fn from(data: ArrayData) -> Self {
- assert_eq!(
- data.data_type(),
- &<OffsetSize as BinaryOffsetSizeTrait>::DATA_TYPE,
- "[Large]BinaryArray expects Datatype::[Large]Binary"
- );
- assert_eq!(
- data.buffers().len(),
- 2,
- "BinaryArray data should contain 2 buffers only (offsets and values)"
- );
- let offsets = data.buffers()[0].as_ptr();
- let values = data.buffers()[1].as_ptr();
- Self {
- data,
- value_offsets: unsafe { RawPtrBox::new(offsets) },
- value_data: unsafe { RawPtrBox::new(values) },
- }
- }
-}
-
-impl<Ptr, OffsetSize: BinaryOffsetSizeTrait> FromIterator<Option<Ptr>>
- for GenericBinaryArray<OffsetSize>
-where
- Ptr: AsRef<[u8]>,
-{
- fn from_iter<I: IntoIterator<Item = Option<Ptr>>>(iter: I) -> Self {
- let iter = iter.into_iter();
- let (_, data_len) = iter.size_hint();
- let data_len = data_len.expect("Iterator must be sized"); // panic if no upper bound.
-
- let mut offsets = Vec::with_capacity(data_len + 1);
- let mut values = Vec::new();
- let mut null_buf = MutableBuffer::new_null(data_len);
- let mut length_so_far: OffsetSize = OffsetSize::zero();
- offsets.push(length_so_far);
-
- {
- let null_slice = null_buf.as_slice_mut();
-
- for (i, s) in iter.enumerate() {
- if let Some(s) = s {
- let s = s.as_ref();
- bit_util::set_bit(null_slice, i);
- length_so_far += OffsetSize::from_usize(s.len()).unwrap();
- values.extend_from_slice(s);
- }
- // always add an element in offsets
- offsets.push(length_so_far);
- }
- }
-
- // calculate actual data_len, which may be different from the iterator's upper bound
- let data_len = offsets.len() - 1;
- let array_data = ArrayData::builder(OffsetSize::DATA_TYPE)
- .len(data_len)
- .add_buffer(Buffer::from_slice_ref(&offsets))
- .add_buffer(Buffer::from_slice_ref(&values))
- .null_bit_buffer(null_buf.into())
- .build();
- Self::from(array_data)
- }
-}
-
-/// An array where each element is a byte whose maximum length is represented by a i32.
-pub type BinaryArray = GenericBinaryArray<i32>;
-
-/// An array where each element is a byte whose maximum length is represented by a i64.
-pub type LargeBinaryArray = GenericBinaryArray<i64>;
-
-impl<'a, T: BinaryOffsetSizeTrait> IntoIterator for &'a GenericBinaryArray<T> {
- type Item = Option<&'a [u8]>;
- type IntoIter = GenericBinaryIter<'a, T>;
-
- fn into_iter(self) -> Self::IntoIter {
- GenericBinaryIter::<'a, T>::new(self)
- }
-}
-
-impl<OffsetSize: BinaryOffsetSizeTrait> From<Vec<Option<&[u8]>>>
- for GenericBinaryArray<OffsetSize>
-{
- fn from(v: Vec<Option<&[u8]>>) -> Self {
- GenericBinaryArray::<OffsetSize>::from_opt_vec(v)
- }
-}
-
-impl<OffsetSize: BinaryOffsetSizeTrait> From<Vec<&[u8]>>
- for GenericBinaryArray<OffsetSize>
-{
- fn from(v: Vec<&[u8]>) -> Self {
- GenericBinaryArray::<OffsetSize>::from_vec(v)
- }
-}
-
-impl<T: BinaryOffsetSizeTrait> From<GenericListArray<T>> for GenericBinaryArray<T> {
- fn from(v: GenericListArray<T>) -> Self {
- GenericBinaryArray::<T>::from_list(v)
- }
-}
-
-/// A type of `FixedSizeListArray` whose elements are binaries.
-pub struct FixedSizeBinaryArray {
- data: ArrayData,
- value_data: RawPtrBox<u8>,
- length: i32,
-}
-
-impl FixedSizeBinaryArray {
- /// Returns the element at index `i` as a byte slice.
- pub fn value(&self, i: usize) -> &[u8] {
- assert!(
- i < self.data.len(),
- "FixedSizeBinaryArray out of bounds access"
- );
- let offset = i.checked_add(self.data.offset()).unwrap();
- unsafe {
- let pos = self.value_offset_at(offset);
- std::slice::from_raw_parts(
- self.value_data.as_ptr().offset(pos as isize),
- (self.value_offset_at(offset + 1) - pos) as usize,
- )
- }
- }
-
- /// Returns the offset for the element at index `i`.
- ///
- /// Note this doesn't do any bound checking, for performance reason.
- #[inline]
- pub fn value_offset(&self, i: usize) -> i32 {
- self.value_offset_at(self.data.offset() + i)
- }
-
- /// Returns the length for an element.
- ///
- /// All elements have the same length as the array is a fixed size.
- #[inline]
- pub fn value_length(&self) -> i32 {
- self.length
- }
-
- /// Returns a clone of the value data buffer
- pub fn value_data(&self) -> Buffer {
- self.data.buffers()[0].clone()
- }
-
- /// Create an array from an iterable argument of sparse byte slices.
- /// Sparsity means that items returned by the iterator are optional, i.e input argument can
- /// contain `None` items.
- ///
- /// # Examles
- ///
- /// ```
- /// use arrow::array::FixedSizeBinaryArray;
- /// let input_arg = vec![
- /// None,
- /// Some(vec![7, 8]),
- /// Some(vec![9, 10]),
- /// None,
- /// Some(vec![13, 14]),
- /// None,
- /// ];
- /// let array = FixedSizeBinaryArray::try_from_sparse_iter(input_arg.into_iter()).unwrap();
- /// ```
- ///
- /// # Errors
- ///
- /// Returns error if argument has length zero, or sizes of nested slices don't match.
- pub fn try_from_sparse_iter<T, U>(mut iter: T) -> Result<Self, ArrowError>
- where
- T: Iterator<Item = Option<U>>,
- U: AsRef<[u8]>,
- {
- let mut len = 0;
- let mut size = None;
- let mut byte = 0;
- let mut null_buf = MutableBuffer::from_len_zeroed(0);
- let mut buffer = MutableBuffer::from_len_zeroed(0);
- let mut prepend = 0;
- iter.try_for_each(|item| -> Result<(), ArrowError> {
- // extend null bitmask by one byte per each 8 items
- if byte == 0 {
- null_buf.push(0u8);
- byte = 8;
- }
- byte -= 1;
-
- if let Some(slice) = item {
- let slice = slice.as_ref();
- if let Some(size) = size {
- if size != slice.len() {
- return Err(ArrowError::InvalidArgumentError(format!(
- "Nested array size mismatch: one is {}, and the other is {}",
- size,
- slice.len()
- )));
- }
- } else {
- size = Some(slice.len());
- buffer.extend_zeros(slice.len() * prepend);
- }
- bit_util::set_bit(null_buf.as_slice_mut(), len);
- buffer.extend_from_slice(slice);
- } else if let Some(size) = size {
- buffer.extend_zeros(size);
- } else {
- prepend += 1;
- }
-
- len += 1;
-
- Ok(())
- })?;
-
- if len == 0 {
- return Err(ArrowError::InvalidArgumentError(
- "Input iterable argument has no data".to_owned(),
- ));
- }
-
- let size = size.unwrap_or(0);
- let array_data = ArrayData::new(
- DataType::FixedSizeBinary(size as i32),
- len,
- None,
- Some(null_buf.into()),
- 0,
- vec![buffer.into()],
- vec![],
- );
- Ok(FixedSizeBinaryArray::from(array_data))
- }
-
- /// Create an array from an iterable argument of byte slices.
- ///
- /// # Examles
- ///
- /// ```
- /// use arrow::array::FixedSizeBinaryArray;
- /// let input_arg = vec![
- /// vec![1, 2],
- /// vec![3, 4],
- /// vec![5, 6],
- /// ];
- /// let array = FixedSizeBinaryArray::try_from_iter(input_arg.into_iter()).unwrap();
- /// ```
- ///
- /// # Errors
- ///
- /// Returns error if argument has length zero, or sizes of nested slices don't match.
- pub fn try_from_iter<T, U>(mut iter: T) -> Result<Self, ArrowError>
- where
- T: Iterator<Item = U>,
- U: AsRef<[u8]>,
- {
- let mut len = 0;
- let mut size = None;
- let mut buffer = MutableBuffer::from_len_zeroed(0);
- iter.try_for_each(|item| -> Result<(), ArrowError> {
- let slice = item.as_ref();
- if let Some(size) = size {
- if size != slice.len() {
- return Err(ArrowError::InvalidArgumentError(format!(
- "Nested array size mismatch: one is {}, and the other is {}",
- size,
- slice.len()
- )));
- }
- } else {
- size = Some(slice.len());
- }
- buffer.extend_from_slice(slice);
-
- len += 1;
-
- Ok(())
- })?;
-
- if len == 0 {
- return Err(ArrowError::InvalidArgumentError(
- "Input iterable argument has no data".to_owned(),
- ));
- }
-
- let size = size.unwrap_or(0);
- let array_data = ArrayData::builder(DataType::FixedSizeBinary(size as i32))
- .len(len)
- .add_buffer(buffer.into())
- .build();
- Ok(FixedSizeBinaryArray::from(array_data))
- }
-
- #[inline]
- fn value_offset_at(&self, i: usize) -> i32 {
- self.length * i as i32
- }
-}
-
-impl From<ArrayData> for FixedSizeBinaryArray {
- fn from(data: ArrayData) -> Self {
- assert_eq!(
- data.buffers().len(),
- 1,
- "FixedSizeBinaryArray data should contain 1 buffer only (values)"
- );
- let value_data = data.buffers()[0].as_ptr();
- let length = match data.data_type() {
- DataType::FixedSizeBinary(len) => *len,
- _ => panic!("Expected data type to be FixedSizeBinary"),
- };
- Self {
- data,
- value_data: unsafe { RawPtrBox::new(value_data) },
- length,
- }
- }
-}
-
-/// Creates a `FixedSizeBinaryArray` from `FixedSizeList<u8>` array
-impl From<FixedSizeListArray> for FixedSizeBinaryArray {
- fn from(v: FixedSizeListArray) -> Self {
- assert_eq!(
- v.data_ref().child_data()[0].child_data().len(),
- 0,
- "FixedSizeBinaryArray can only be created from list array of u8 values \
- (i.e. FixedSizeList<PrimitiveArray<u8>>)."
- );
- assert_eq!(
- v.data_ref().child_data()[0].data_type(),
- &DataType::UInt8,
- "FixedSizeBinaryArray can only be created from FixedSizeList<u8> arrays, mismatched data types."
- );
-
- let mut builder = ArrayData::builder(DataType::FixedSizeBinary(v.value_length()))
- .len(v.len())
- .add_buffer(v.data_ref().child_data()[0].buffers()[0].clone());
- if let Some(bitmap) = v.data_ref().null_bitmap() {
- builder = builder.null_bit_buffer(bitmap.bits.clone())
- }
-
- let data = builder.build();
- Self::from(data)
- }
-}
-
-impl fmt::Debug for FixedSizeBinaryArray {
- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- write!(f, "FixedSizeBinaryArray<{}>\n[\n", self.value_length())?;
- print_long_array(self, f, |array, index, f| {
- fmt::Debug::fmt(&array.value(index), f)
- })?;
- write!(f, "]")
- }
-}
-
-impl Array for FixedSizeBinaryArray {
- fn as_any(&self) -> &Any {
- self
- }
-
- fn data(&self) -> &ArrayData {
- &self.data
- }
-
- /// Returns the total number of bytes of memory occupied by the buffers owned by this [FixedSizeBinaryArray].
- fn get_buffer_memory_size(&self) -> usize {
- self.data.get_buffer_memory_size()
- }
-
- /// Returns the total number of bytes of memory occupied physically by this [FixedSizeBinaryArray].
- fn get_array_memory_size(&self) -> usize {
- self.data.get_array_memory_size() + mem::size_of_val(self)
- }
-}
-
-/// A type of `DecimalArray` whose elements are binaries.
-pub struct DecimalArray {
- data: ArrayData,
- value_data: RawPtrBox<u8>,
- precision: usize,
- scale: usize,
- length: i32,
-}
-
-impl DecimalArray {
- /// Returns the element at index `i` as i128.
- pub fn value(&self, i: usize) -> i128 {
- assert!(i < self.data.len(), "DecimalArray out of bounds access");
- let offset = i.checked_add(self.data.offset()).unwrap();
- let raw_val = unsafe {
- let pos = self.value_offset_at(offset);
- std::slice::from_raw_parts(
- self.value_data.as_ptr().offset(pos as isize),
- (self.value_offset_at(offset + 1) - pos) as usize,
- )
- };
- let as_array = raw_val.try_into();
- match as_array {
- Ok(v) if raw_val.len() == 16 => i128::from_le_bytes(v),
- _ => panic!("DecimalArray elements are not 128bit integers."),
- }
- }
-
- /// Returns the offset for the element at index `i`.
- ///
- /// Note this doesn't do any bound checking, for performance reason.
- #[inline]
- pub fn value_offset(&self, i: usize) -> i32 {
- self.value_offset_at(self.data.offset() + i)
- }
-
- /// Returns the length for an element.
- ///
- /// All elements have the same length as the array is a fixed size.
- #[inline]
- pub fn value_length(&self) -> i32 {
- self.length
- }
-
- /// Returns a clone of the value data buffer
- pub fn value_data(&self) -> Buffer {
- self.data.buffers()[0].clone()
- }
-
- #[inline]
- fn value_offset_at(&self, i: usize) -> i32 {
- self.length * i as i32
- }
-
- pub fn from_fixed_size_list_array(
- v: FixedSizeListArray,
- precision: usize,
- scale: usize,
- ) -> Self {
- assert_eq!(
- v.data_ref().child_data()[0].child_data().len(),
- 0,
- "DecimalArray can only be created from list array of u8 values \
- (i.e. FixedSizeList<PrimitiveArray<u8>>)."
- );
- assert_eq!(
- v.data_ref().child_data()[0].data_type(),
- &DataType::UInt8,
- "DecimalArray can only be created from FixedSizeList<u8> arrays, mismatched data types."
- );
-
- let mut builder = ArrayData::builder(DataType::Decimal(precision, scale))
- .len(v.len())
- .add_buffer(v.data_ref().child_data()[0].buffers()[0].clone());
- if let Some(bitmap) = v.data_ref().null_bitmap() {
- builder = builder.null_bit_buffer(bitmap.bits.clone())
- }
-
- let data = builder.build();
- Self::from(data)
- }
- pub fn precision(&self) -> usize {
- self.precision
- }
-
- pub fn scale(&self) -> usize {
- self.scale
- }
-}
-
-impl From<ArrayData> for DecimalArray {
- fn from(data: ArrayData) -> Self {
- assert_eq!(
- data.buffers().len(),
- 1,
- "DecimalArray data should contain 1 buffer only (values)"
- );
- let values = data.buffers()[0].as_ptr();
- let (precision, scale) = match data.data_type() {
- DataType::Decimal(precision, scale) => (*precision, *scale),
- _ => panic!("Expected data type to be Decimal"),
- };
- let length = 16;
- Self {
- data,
- value_data: unsafe { RawPtrBox::new(values) },
- precision,
- scale,
- length,
- }
- }
-}
-
-impl fmt::Debug for DecimalArray {
- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- write!(f, "DecimalArray<{}, {}>\n[\n", self.precision, self.scale)?;
- print_long_array(self, f, |array, index, f| {
- fmt::Debug::fmt(&array.value(index), f)
- })?;
- write!(f, "]")
- }
-}
-
-impl Array for DecimalArray {
- fn as_any(&self) -> &Any {
- self
- }
-
- fn data(&self) -> &ArrayData {
- &self.data
- }
-
- /// Returns the total number of bytes of memory occupied by the buffers owned by this [DecimalArray].
- fn get_buffer_memory_size(&self) -> usize {
- self.data.get_buffer_memory_size()
- }
-
- /// Returns the total number of bytes of memory occupied physically by this [DecimalArray].
- fn get_array_memory_size(&self) -> usize {
- self.data.get_array_memory_size() + mem::size_of_val(self)
- }
-}
-
-#[cfg(test)]
-mod tests {
- use crate::{
- array::{LargeListArray, ListArray},
- datatypes::Field,
- };
-
- use super::*;
-
- #[test]
- fn test_binary_array() {
- let values: [u8; 12] = [
- b'h', b'e', b'l', b'l', b'o', b'p', b'a', b'r', b'q', b'u', b'e', b't',
- ];
- let offsets: [i32; 4] = [0, 5, 5, 12];
-
- // Array data: ["hello", "", "parquet"]
- let array_data = ArrayData::builder(DataType::Binary)
- .len(3)
- .add_buffer(Buffer::from_slice_ref(&offsets))
- .add_buffer(Buffer::from_slice_ref(&values))
- .build();
- let binary_array = BinaryArray::from(array_data);
- assert_eq!(3, binary_array.len());
- assert_eq!(0, binary_array.null_count());
- assert_eq!([b'h', b'e', b'l', b'l', b'o'], binary_array.value(0));
- assert_eq!([b'h', b'e', b'l', b'l', b'o'], unsafe {
- binary_array.value_unchecked(0)
- });
- assert_eq!([] as [u8; 0], binary_array.value(1));
- assert_eq!([] as [u8; 0], unsafe { binary_array.value_unchecked(1) });
- assert_eq!(
- [b'p', b'a', b'r', b'q', b'u', b'e', b't'],
- binary_array.value(2)
- );
- assert_eq!([b'p', b'a', b'r', b'q', b'u', b'e', b't'], unsafe {
- binary_array.value_unchecked(2)
- });
- assert_eq!(5, binary_array.value_offsets()[2]);
- assert_eq!(7, binary_array.value_length(2));
- for i in 0..3 {
- assert!(binary_array.is_valid(i));
- assert!(!binary_array.is_null(i));
- }
-
- // Test binary array with offset
- let array_data = ArrayData::builder(DataType::Binary)
- .len(4)
- .offset(1)
- .add_buffer(Buffer::from_slice_ref(&offsets))
- .add_buffer(Buffer::from_slice_ref(&values))
- .build();
- let binary_array = BinaryArray::from(array_data);
- assert_eq!(
- [b'p', b'a', b'r', b'q', b'u', b'e', b't'],
- binary_array.value(1)
- );
- assert_eq!(5, binary_array.value_offsets()[0]);
- assert_eq!(0, binary_array.value_length(0));
- assert_eq!(5, binary_array.value_offsets()[1]);
- assert_eq!(7, binary_array.value_length(1));
- }
-
- #[test]
- fn test_large_binary_array() {
- let values: [u8; 12] = [
- b'h', b'e', b'l', b'l', b'o', b'p', b'a', b'r', b'q', b'u', b'e', b't',
- ];
- let offsets: [i64; 4] = [0, 5, 5, 12];
-
- // Array data: ["hello", "", "parquet"]
- let array_data = ArrayData::builder(DataType::LargeBinary)
- .len(3)
- .add_buffer(Buffer::from_slice_ref(&offsets))
- .add_buffer(Buffer::from_slice_ref(&values))
- .build();
- let binary_array = LargeBinaryArray::from(array_data);
- assert_eq!(3, binary_array.len());
- assert_eq!(0, binary_array.null_count());
- assert_eq!([b'h', b'e', b'l', b'l', b'o'], binary_array.value(0));
- assert_eq!([b'h', b'e', b'l', b'l', b'o'], unsafe {
- binary_array.value_unchecked(0)
- });
- assert_eq!([] as [u8; 0], binary_array.value(1));
- assert_eq!([] as [u8; 0], unsafe { binary_array.value_unchecked(1) });
- assert_eq!(
- [b'p', b'a', b'r', b'q', b'u', b'e', b't'],
- binary_array.value(2)
- );
- assert_eq!([b'p', b'a', b'r', b'q', b'u', b'e', b't'], unsafe {
- binary_array.value_unchecked(2)
- });
- assert_eq!(5, binary_array.value_offsets()[2]);
- assert_eq!(7, binary_array.value_length(2));
- for i in 0..3 {
- assert!(binary_array.is_valid(i));
- assert!(!binary_array.is_null(i));
- }
-
- // Test binary array with offset
- let array_data = ArrayData::builder(DataType::LargeBinary)
- .len(4)
- .offset(1)
- .add_buffer(Buffer::from_slice_ref(&offsets))
- .add_buffer(Buffer::from_slice_ref(&values))
- .build();
- let binary_array = LargeBinaryArray::from(array_data);
- assert_eq!(
- [b'p', b'a', b'r', b'q', b'u', b'e', b't'],
- binary_array.value(1)
- );
- assert_eq!([b'p', b'a', b'r', b'q', b'u', b'e', b't'], unsafe {
- binary_array.value_unchecked(1)
- });
- assert_eq!(5, binary_array.value_offsets()[0]);
- assert_eq!(0, binary_array.value_length(0));
- assert_eq!(5, binary_array.value_offsets()[1]);
- assert_eq!(7, binary_array.value_length(1));
- }
-
- #[test]
- fn test_binary_array_from_list_array() {
- let values: [u8; 12] = [
- b'h', b'e', b'l', b'l', b'o', b'p', b'a', b'r', b'q', b'u', b'e', b't',
- ];
- let values_data = ArrayData::builder(DataType::UInt8)
- .len(12)
- .add_buffer(Buffer::from(&values[..]))
- .build();
- let offsets: [i32; 4] = [0, 5, 5, 12];
-
- // Array data: ["hello", "", "parquet"]
- let array_data1 = ArrayData::builder(DataType::Binary)
- .len(3)
- .add_buffer(Buffer::from_slice_ref(&offsets))
- .add_buffer(Buffer::from_slice_ref(&values))
- .build();
- let binary_array1 = BinaryArray::from(array_data1);
-
- let data_type =
- DataType::List(Box::new(Field::new("item", DataType::UInt8, false)));
- let array_data2 = ArrayData::builder(data_type)
- .len(3)
- .add_buffer(Buffer::from_slice_ref(&offsets))
- .add_child_data(values_data)
- .build();
- let list_array = ListArray::from(array_data2);
- let binary_array2 = BinaryArray::from(list_array);
-
- assert_eq!(2, binary_array2.data().buffers().len());
- assert_eq!(0, binary_array2.data().child_data().len());
-
- assert_eq!(binary_array1.len(), binary_array2.len());
- assert_eq!(binary_array1.null_count(), binary_array2.null_count());
- assert_eq!(binary_array1.value_offsets(), binary_array2.value_offsets());
- for i in 0..binary_array1.len() {
- assert_eq!(binary_array1.value(i), binary_array2.value(i));
- assert_eq!(binary_array1.value(i), unsafe {
- binary_array2.value_unchecked(i)
- });
- assert_eq!(binary_array1.value_length(i), binary_array2.value_length(i));
- }
- }
-
- #[test]
- fn test_large_binary_array_from_list_array() {
- let values: [u8; 12] = [
- b'h', b'e', b'l', b'l', b'o', b'p', b'a', b'r', b'q', b'u', b'e', b't',
- ];
- let values_data = ArrayData::builder(DataType::UInt8)
- .len(12)
- .add_buffer(Buffer::from(&values[..]))
- .build();
- let offsets: [i64; 4] = [0, 5, 5, 12];
-
- // Array data: ["hello", "", "parquet"]
- let array_data1 = ArrayData::builder(DataType::LargeBinary)
- .len(3)
- .add_buffer(Buffer::from_slice_ref(&offsets))
- .add_buffer(Buffer::from_slice_ref(&values))
- .build();
- let binary_array1 = LargeBinaryArray::from(array_data1);
-
- let data_type =
- DataType::LargeList(Box::new(Field::new("item", DataType::UInt8, false)));
- let array_data2 = ArrayData::builder(data_type)
- .len(3)
- .add_buffer(Buffer::from_slice_ref(&offsets))
- .add_child_data(values_data)
- .build();
- let list_array = LargeListArray::from(array_data2);
- let binary_array2 = LargeBinaryArray::from(list_array);
-
- assert_eq!(2, binary_array2.data().buffers().len());
- assert_eq!(0, binary_array2.data().child_data().len());
-
- assert_eq!(binary_array1.len(), binary_array2.len());
- assert_eq!(binary_array1.null_count(), binary_array2.null_count());
- assert_eq!(binary_array1.value_offsets(), binary_array2.value_offsets());
- for i in 0..binary_array1.len() {
- assert_eq!(binary_array1.value(i), binary_array2.value(i));
- assert_eq!(binary_array1.value(i), unsafe {
- binary_array2.value_unchecked(i)
- });
- assert_eq!(binary_array1.value_length(i), binary_array2.value_length(i));
- }
- }
-
- fn test_generic_binary_array_from_opt_vec<T: BinaryOffsetSizeTrait>() {
- let values: Vec<Option<&[u8]>> =
- vec![Some(b"one"), Some(b"two"), None, Some(b""), Some(b"three")];
- let array = GenericBinaryArray::<T>::from_opt_vec(values);
- assert_eq!(array.len(), 5);
- assert_eq!(array.value(0), b"one");
- assert_eq!(array.value(1), b"two");
- assert_eq!(array.value(3), b"");
- assert_eq!(array.value(4), b"three");
- assert_eq!(array.is_null(0), false);
- assert_eq!(array.is_null(1), false);
- assert_eq!(array.is_null(2), true);
- assert_eq!(array.is_null(3), false);
- assert_eq!(array.is_null(4), false);
- }
-
- #[test]
- fn test_large_binary_array_from_opt_vec() {
- test_generic_binary_array_from_opt_vec::<i64>()
- }
-
- #[test]
- fn test_binary_array_from_opt_vec() {
- test_generic_binary_array_from_opt_vec::<i32>()
- }
-
- #[test]
- fn test_binary_array_from_unbound_iter() {
- // iterator that doesn't declare (upper) size bound
- let value_iter = (0..)
- .scan(0usize, |pos, i| {
- if *pos < 10 {
- *pos += 1;
- Some(Some(format!("value {}", i)))
- } else {
- // actually returns up to 10 values
- None
- }
- })
- // limited using take()
- .take(100);
-
- let (_, upper_size_bound) = value_iter.size_hint();
- // the upper bound, defined by take above, is 100
- assert_eq!(upper_size_bound, Some(100));
- let binary_array: BinaryArray = value_iter.collect();
- // but the actual number of items in the array should be 10
- assert_eq!(binary_array.len(), 10);
- }
-
- #[test]
- #[should_panic(
- expected = "assertion failed: `(left == right)`\n left: `UInt32`,\n \
- right: `UInt8`: BinaryArray can only be created from List<u8> arrays, \
- mismatched data types."
- )]
- fn test_binary_array_from_incorrect_list_array() {
- let values: [u32; 12] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11];
- let values_data = ArrayData::builder(DataType::UInt32)
- .len(12)
- .add_buffer(Buffer::from_slice_ref(&values))
- .build();
- let offsets: [i32; 4] = [0, 5, 5, 12];
-
- let data_type =
- DataType::List(Box::new(Field::new("item", DataType::UInt32, false)));
- let array_data = ArrayData::builder(data_type)
- .len(3)
- .add_buffer(Buffer::from_slice_ref(&offsets))
- .add_child_data(values_data)
- .build();
- let list_array = ListArray::from(array_data);
- BinaryArray::from(list_array);
- }
-
- #[test]
- fn test_fixed_size_binary_array() {
- let values: [u8; 15] = *b"hellotherearrow";
-
- let array_data = ArrayData::builder(DataType::FixedSizeBinary(5))
- .len(3)
- .add_buffer(Buffer::from(&values[..]))
- .build();
- let fixed_size_binary_array = FixedSizeBinaryArray::from(array_data);
- assert_eq!(3, fixed_size_binary_array.len());
- assert_eq!(0, fixed_size_binary_array.null_count());
- assert_eq!(
- [b'h', b'e', b'l', b'l', b'o'],
- fixed_size_binary_array.value(0)
- );
- assert_eq!(
- [b't', b'h', b'e', b'r', b'e'],
- fixed_size_binary_array.value(1)
- );
- assert_eq!(
- [b'a', b'r', b'r', b'o', b'w'],
- fixed_size_binary_array.value(2)
- );
- assert_eq!(5, fixed_size_binary_array.value_length());
- assert_eq!(10, fixed_size_binary_array.value_offset(2));
- for i in 0..3 {
- assert!(fixed_size_binary_array.is_valid(i));
- assert!(!fixed_size_binary_array.is_null(i));
- }
-
- // Test binary array with offset
- let array_data = ArrayData::builder(DataType::FixedSizeBinary(5))
- .len(2)
- .offset(1)
- .add_buffer(Buffer::from(&values[..]))
- .build();
- let fixed_size_binary_array = FixedSizeBinaryArray::from(array_data);
- assert_eq!(
- [b't', b'h', b'e', b'r', b'e'],
- fixed_size_binary_array.value(0)
- );
- assert_eq!(
- [b'a', b'r', b'r', b'o', b'w'],
- fixed_size_binary_array.value(1)
- );
- assert_eq!(2, fixed_size_binary_array.len());
- assert_eq!(5, fixed_size_binary_array.value_offset(0));
- assert_eq!(5, fixed_size_binary_array.value_length());
- assert_eq!(10, fixed_size_binary_array.value_offset(1));
- }
-
- #[test]
- #[should_panic(
- expected = "FixedSizeBinaryArray can only be created from list array of u8 values \
- (i.e. FixedSizeList<PrimitiveArray<u8>>)."
- )]
- fn test_fixed_size_binary_array_from_incorrect_list_array() {
- let values: [u32; 12] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11];
- let values_data = ArrayData::builder(DataType::UInt32)
- .len(12)
- .add_buffer(Buffer::from_slice_ref(&values))
- .add_child_data(ArrayData::builder(DataType::Boolean).build())
- .build();
-
- let array_data = ArrayData::builder(DataType::FixedSizeList(
- Box::new(Field::new("item", DataType::Binary, false)),
- 4,
- ))
- .len(3)
- .add_child_data(values_data)
- .build();
- let list_array = FixedSizeListArray::from(array_data);
- FixedSizeBinaryArray::from(list_array);
- }
-
- #[test]
- #[should_panic(expected = "BinaryArray out of bounds access")]
- fn test_binary_array_get_value_index_out_of_bound() {
- let values: [u8; 12] =
- [104, 101, 108, 108, 111, 112, 97, 114, 113, 117, 101, 116];
- let offsets: [i32; 4] = [0, 5, 5, 12];
- let array_data = ArrayData::builder(DataType::Binary)
- .len(3)
- .add_buffer(Buffer::from_slice_ref(&offsets))
- .add_buffer(Buffer::from_slice_ref(&values))
- .build();
- let binary_array = BinaryArray::from(array_data);
- binary_array.value(4);
- }
-
- #[test]
- fn test_binary_array_fmt_debug() {
- let values: [u8; 15] = *b"hellotherearrow";
-
- let array_data = ArrayData::builder(DataType::FixedSizeBinary(5))
- .len(3)
- .add_buffer(Buffer::from(&values[..]))
- .build();
- let arr = FixedSizeBinaryArray::from(array_data);
- assert_eq!(
- "FixedSizeBinaryArray<5>\n[\n [104, 101, 108, 108, 111],\n [116, 104, 101, 114, 101],\n [97, 114, 114, 111, 119],\n]",
- format!("{:?}", arr)
- );
- }
-
- #[test]
- fn test_decimal_array() {
- // let val_8887: [u8; 16] = [192, 219, 180, 17, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
- // let val_neg_8887: [u8; 16] = [64, 36, 75, 238, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255];
- let values: [u8; 32] = [
- 192, 219, 180, 17, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 36, 75, 238, 253,
- 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
- ];
- let array_data = ArrayData::builder(DataType::Decimal(23, 6))
- .len(2)
- .add_buffer(Buffer::from(&values[..]))
- .build();
- let decimal_array = DecimalArray::from(array_data);
- assert_eq!(8_887_000_000, decimal_array.value(0));
- assert_eq!(-8_887_000_000, decimal_array.value(1));
- assert_eq!(16, decimal_array.value_length());
- }
-
- #[test]
- fn test_decimal_array_fmt_debug() {
- let values: [u8; 32] = [
- 192, 219, 180, 17, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 36, 75, 238, 253,
- 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
- ];
- let array_data = ArrayData::builder(DataType::Decimal(23, 6))
- .len(2)
- .add_buffer(Buffer::from(&values[..]))
- .build();
- let arr = DecimalArray::from(array_data);
- assert_eq!(
- "DecimalArray<23, 6>\n[\n 8887000000,\n -8887000000,\n]",
- format!("{:?}", arr)
- );
- }
-}
diff --git a/rust/arrow/src/array/array_boolean.rs b/rust/arrow/src/array/array_boolean.rs
deleted file mode 100644
index 67af85d..0000000
--- a/rust/arrow/src/array/array_boolean.rs
+++ /dev/null
@@ -1,291 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::borrow::Borrow;
-use std::convert::From;
-use std::iter::{FromIterator, IntoIterator};
-use std::mem;
-use std::{any::Any, fmt};
-
-use super::*;
-use super::{array::print_long_array, raw_pointer::RawPtrBox};
-use crate::buffer::{Buffer, MutableBuffer};
-use crate::util::bit_util;
-
-/// Array of bools
-pub struct BooleanArray {
- data: ArrayData,
- /// Pointer to the value array. The lifetime of this must be <= to the value buffer
- /// stored in `data`, so it's safe to store.
- raw_values: RawPtrBox<u8>,
-}
-
-impl fmt::Debug for BooleanArray {
- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- write!(f, "BooleanArray\n[\n")?;
- print_long_array(self, f, |array, index, f| {
- fmt::Debug::fmt(&array.value(index), f)
- })?;
- write!(f, "]")
- }
-}
-
-impl BooleanArray {
- /// Returns the length of this array.
- pub fn len(&self) -> usize {
- self.data.len()
- }
-
- /// Returns whether this array is empty.
- pub fn is_empty(&self) -> bool {
- self.data.is_empty()
- }
-
- // Returns a new boolean array builder
- pub fn builder(capacity: usize) -> BooleanBuilder {
- BooleanBuilder::new(capacity)
- }
-
- /// Returns a `Buffer` holding all the values of this array.
- ///
- /// Note this doesn't take the offset of this array into account.
- pub fn values(&self) -> &Buffer {
- &self.data.buffers()[0]
- }
-
- /// Returns the boolean value at index `i`.
- ///
- /// # Safety
- /// This doesn't check bounds, the caller must ensure that index < self.len()
- pub unsafe fn value_unchecked(&self, i: usize) -> bool {
- let offset = i + self.offset();
- bit_util::get_bit_raw(self.raw_values.as_ptr(), offset)
- }
-
- /// Returns the boolean value at index `i`.
- ///
- /// Note this doesn't do any bound checking, for performance reason.
- pub fn value(&self, i: usize) -> bool {
- debug_assert!(i < self.len());
- unsafe { self.value_unchecked(i) }
- }
-}
-
-impl Array for BooleanArray {
- fn as_any(&self) -> &Any {
- self
- }
-
- fn data(&self) -> &ArrayData {
- &self.data
- }
-
- /// Returns the total number of bytes of memory occupied by the buffers owned by this [BooleanArray].
- fn get_buffer_memory_size(&self) -> usize {
- self.data.get_buffer_memory_size()
- }
-
- /// Returns the total number of bytes of memory occupied physically by this [BooleanArray].
- fn get_array_memory_size(&self) -> usize {
- self.data.get_array_memory_size() + mem::size_of_val(self)
- }
-}
-
-impl From<Vec<bool>> for BooleanArray {
- fn from(data: Vec<bool>) -> Self {
- let mut mut_buf = MutableBuffer::new_null(data.len());
- {
- let mut_slice = mut_buf.as_slice_mut();
- for (i, b) in data.iter().enumerate() {
- if *b {
- bit_util::set_bit(mut_slice, i);
- }
- }
- }
- let array_data = ArrayData::builder(DataType::Boolean)
- .len(data.len())
- .add_buffer(mut_buf.into())
- .build();
- BooleanArray::from(array_data)
- }
-}
-
-impl From<Vec<Option<bool>>> for BooleanArray {
- fn from(data: Vec<Option<bool>>) -> Self {
- BooleanArray::from_iter(data.iter())
- }
-}
-
-impl From<ArrayData> for BooleanArray {
- fn from(data: ArrayData) -> Self {
- assert_eq!(
- data.buffers().len(),
- 1,
- "BooleanArray data should contain a single buffer only (values buffer)"
- );
- let ptr = data.buffers()[0].as_ptr();
- Self {
- data,
- raw_values: unsafe { RawPtrBox::new(ptr) },
- }
- }
-}
-
-impl<'a> IntoIterator for &'a BooleanArray {
- type Item = Option<bool>;
- type IntoIter = BooleanIter<'a>;
-
- fn into_iter(self) -> Self::IntoIter {
- BooleanIter::<'a>::new(self)
- }
-}
-
-impl<'a> BooleanArray {
- /// constructs a new iterator
- pub fn iter(&'a self) -> BooleanIter<'a> {
- BooleanIter::<'a>::new(&self)
- }
-}
-
-impl<Ptr: Borrow<Option<bool>>> FromIterator<Ptr> for BooleanArray {
- fn from_iter<I: IntoIterator<Item = Ptr>>(iter: I) -> Self {
- let iter = iter.into_iter();
- let (_, data_len) = iter.size_hint();
- let data_len = data_len.expect("Iterator must be sized"); // panic if no upper bound.
-
- let num_bytes = bit_util::ceil(data_len, 8);
- let mut null_buf = MutableBuffer::from_len_zeroed(num_bytes);
- let mut val_buf = MutableBuffer::from_len_zeroed(num_bytes);
-
- let data = val_buf.as_slice_mut();
-
- let null_slice = null_buf.as_slice_mut();
- iter.enumerate().for_each(|(i, item)| {
- if let Some(a) = item.borrow() {
- bit_util::set_bit(null_slice, i);
- if *a {
- bit_util::set_bit(data, i);
- }
- }
- });
-
- let data = ArrayData::new(
- DataType::Boolean,
- data_len,
- None,
- Some(null_buf.into()),
- 0,
- vec![val_buf.into()],
- vec![],
- );
- BooleanArray::from(data)
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- use crate::buffer::Buffer;
- use crate::datatypes::DataType;
-
- #[test]
- fn test_boolean_fmt_debug() {
- let arr = BooleanArray::from(vec![true, false, false]);
- assert_eq!(
- "BooleanArray\n[\n true,\n false,\n false,\n]",
- format!("{:?}", arr)
- );
- }
-
- #[test]
- fn test_boolean_with_null_fmt_debug() {
- let mut builder = BooleanArray::builder(3);
- builder.append_value(true).unwrap();
- builder.append_null().unwrap();
- builder.append_value(false).unwrap();
- let arr = builder.finish();
- assert_eq!(
- "BooleanArray\n[\n true,\n null,\n false,\n]",
- format!("{:?}", arr)
- );
- }
-
- #[test]
- fn test_boolean_array_from_vec() {
- let buf = Buffer::from([10_u8]);
- let arr = BooleanArray::from(vec![false, true, false, true]);
- assert_eq!(&buf, arr.values());
- assert_eq!(4, arr.len());
- assert_eq!(0, arr.offset());
- assert_eq!(0, arr.null_count());
- for i in 0..4 {
- assert!(!arr.is_null(i));
- assert!(arr.is_valid(i));
- assert_eq!(i == 1 || i == 3, arr.value(i), "failed at {}", i)
- }
- }
-
- #[test]
- fn test_boolean_array_from_vec_option() {
- let buf = Buffer::from([10_u8]);
- let arr = BooleanArray::from(vec![Some(false), Some(true), None, Some(true)]);
- assert_eq!(&buf, arr.values());
- assert_eq!(4, arr.len());
- assert_eq!(0, arr.offset());
- assert_eq!(1, arr.null_count());
- for i in 0..4 {
- if i == 2 {
- assert!(arr.is_null(i));
- assert!(!arr.is_valid(i));
- } else {
- assert!(!arr.is_null(i));
- assert!(arr.is_valid(i));
- assert_eq!(i == 1 || i == 3, arr.value(i), "failed at {}", i)
- }
- }
- }
-
- #[test]
- fn test_boolean_array_builder() {
- // Test building a boolean array with ArrayData builder and offset
- // 000011011
- let buf = Buffer::from([27_u8]);
- let buf2 = buf.clone();
- let data = ArrayData::builder(DataType::Boolean)
- .len(5)
- .offset(2)
- .add_buffer(buf)
- .build();
- let arr = BooleanArray::from(data);
- assert_eq!(&buf2, arr.values());
- assert_eq!(5, arr.len());
- assert_eq!(2, arr.offset());
- assert_eq!(0, arr.null_count());
- for i in 0..3 {
- assert_eq!(i != 0, arr.value(i), "failed at {}", i);
- }
- }
-
- #[test]
- #[should_panic(expected = "BooleanArray data should contain a single buffer only \
- (values buffer)")]
- fn test_boolean_array_invalid_buffer_len() {
- let data = ArrayData::builder(DataType::Boolean).len(5).build();
- BooleanArray::from(data);
- }
-}
diff --git a/rust/arrow/src/array/array_dictionary.rs b/rust/arrow/src/array/array_dictionary.rs
deleted file mode 100644
index 5948658..0000000
--- a/rust/arrow/src/array/array_dictionary.rs
+++ /dev/null
@@ -1,408 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::any::Any;
-use std::fmt;
-use std::iter::IntoIterator;
-use std::mem;
-use std::{convert::From, iter::FromIterator};
-
-use super::{
- make_array, Array, ArrayData, ArrayRef, PrimitiveArray, PrimitiveBuilder,
- StringArray, StringBuilder, StringDictionaryBuilder,
-};
-use crate::datatypes::ArrowNativeType;
-use crate::datatypes::{ArrowDictionaryKeyType, ArrowPrimitiveType, DataType};
-
-/// A dictionary array where each element is a single value indexed by an integer key.
-/// This is mostly used to represent strings or a limited set of primitive types as integers,
-/// for example when doing NLP analysis or representing chromosomes by name.
-///
-/// Example **with nullable** data:
-///
-/// ```
-/// use arrow::array::{DictionaryArray, Int8Array};
-/// use arrow::datatypes::Int8Type;
-/// let test = vec!["a", "a", "b", "c"];
-/// let array : DictionaryArray<Int8Type> = test.iter().map(|&x| if x == "b" {None} else {Some(x)}).collect();
-/// assert_eq!(array.keys(), &Int8Array::from(vec![Some(0), Some(0), None, Some(1)]));
-/// ```
-///
-/// Example **without nullable** data:
-///
-/// ```
-/// use arrow::array::{DictionaryArray, Int8Array};
-/// use arrow::datatypes::Int8Type;
-/// let test = vec!["a", "a", "b", "c"];
-/// let array : DictionaryArray<Int8Type> = test.into_iter().collect();
-/// assert_eq!(array.keys(), &Int8Array::from(vec![0, 0, 1, 2]));
-/// ```
-pub struct DictionaryArray<K: ArrowPrimitiveType> {
- /// Data of this dictionary. Note that this is _not_ compatible with the C Data interface,
- /// as, in the current implementation, `values` below are the first child of this struct.
- data: ArrayData,
-
- /// The keys of this dictionary. These are constructed from the buffer and null bitmap
- /// of `data`.
- /// Also, note that these do not correspond to the true values of this array. Rather, they map
- /// to the real values.
- keys: PrimitiveArray<K>,
-
- /// Array of dictionary values (can by any DataType).
- values: ArrayRef,
-
- /// Values are ordered.
- is_ordered: bool,
-}
-
-impl<'a, K: ArrowPrimitiveType> DictionaryArray<K> {
- /// Return an iterator to the keys of this dictionary.
- pub fn keys(&self) -> &PrimitiveArray<K> {
- &self.keys
- }
-
- /// Returns an array view of the keys of this dictionary
- pub fn keys_array(&self) -> PrimitiveArray<K> {
- let data = self.data_ref();
- let keys_data = ArrayData::new(
- K::DATA_TYPE,
- data.len(),
- Some(data.null_count()),
- data.null_buffer().cloned(),
- data.offset(),
- data.buffers().to_vec(),
- vec![],
- );
- PrimitiveArray::<K>::from(keys_data)
- }
-
- /// Returns the lookup key by doing reverse dictionary lookup
- pub fn lookup_key(&self, value: &str) -> Option<K::Native> {
- let rd_buf: &StringArray =
- self.values.as_any().downcast_ref::<StringArray>().unwrap();
-
- (0..rd_buf.len())
- .position(|i| rd_buf.value(i) == value)
- .map(K::Native::from_usize)
- .flatten()
- }
-
- /// Returns an `ArrayRef` to the dictionary values.
- pub fn values(&self) -> ArrayRef {
- self.values.clone()
- }
-
- /// Returns a clone of the value type of this list.
- pub fn value_type(&self) -> DataType {
- self.values.data_ref().data_type().clone()
- }
-
- /// The length of the dictionary is the length of the keys array.
- pub fn len(&self) -> usize {
- self.keys.len()
- }
-
- /// Whether this dictionary is empty
- pub fn is_empty(&self) -> bool {
- self.keys.is_empty()
- }
-
- // Currently exists for compatibility purposes with Arrow IPC.
- pub fn is_ordered(&self) -> bool {
- self.is_ordered
- }
-}
-
-/// Constructs a `DictionaryArray` from an array data reference.
-impl<T: ArrowPrimitiveType> From<ArrayData> for DictionaryArray<T> {
- fn from(data: ArrayData) -> Self {
- assert_eq!(
- data.buffers().len(),
- 1,
- "DictionaryArray data should contain a single buffer only (keys)."
- );
- assert_eq!(
- data.child_data().len(),
- 1,
- "DictionaryArray should contain a single child array (values)."
- );
-
- if let DataType::Dictionary(key_data_type, _) = data.data_type() {
- if key_data_type.as_ref() != &T::DATA_TYPE {
- panic!("DictionaryArray's data type must match.")
- };
- // create a zero-copy of the keys' data
- let keys = PrimitiveArray::<T>::from(ArrayData::new(
- T::DATA_TYPE,
- data.len(),
- Some(data.null_count()),
- data.null_buffer().cloned(),
- data.offset(),
- data.buffers().to_vec(),
- vec![],
- ));
- let values = make_array(data.child_data()[0].clone());
- Self {
- data,
- keys,
- values,
- is_ordered: false,
- }
- } else {
- panic!("DictionaryArray must have Dictionary data type.")
- }
- }
-}
-
-/// Constructs a `DictionaryArray` from an iterator of optional strings.
-impl<'a, T: ArrowPrimitiveType + ArrowDictionaryKeyType> FromIterator<Option<&'a str>>
- for DictionaryArray<T>
-{
- fn from_iter<I: IntoIterator<Item = Option<&'a str>>>(iter: I) -> Self {
- let it = iter.into_iter();
- let (lower, _) = it.size_hint();
- let key_builder = PrimitiveBuilder::<T>::new(lower);
- let value_builder = StringBuilder::new(256);
- let mut builder = StringDictionaryBuilder::new(key_builder, value_builder);
- it.for_each(|i| {
- if let Some(i) = i {
- // Note: impl ... for Result<DictionaryArray<T>> fails with
- // error[E0117]: only traits defined in the current crate can be implemented for arbitrary types
- builder
- .append(i)
- .expect("Unable to append a value to a dictionary array.");
- } else {
- builder
- .append_null()
- .expect("Unable to append a null value to a dictionary array.");
- }
- });
-
- builder.finish()
- }
-}
-
-/// Constructs a `DictionaryArray` from an iterator of strings.
-impl<'a, T: ArrowPrimitiveType + ArrowDictionaryKeyType> FromIterator<&'a str>
- for DictionaryArray<T>
-{
- fn from_iter<I: IntoIterator<Item = &'a str>>(iter: I) -> Self {
- let it = iter.into_iter();
- let (lower, _) = it.size_hint();
- let key_builder = PrimitiveBuilder::<T>::new(lower);
- let value_builder = StringBuilder::new(256);
- let mut builder = StringDictionaryBuilder::new(key_builder, value_builder);
- it.for_each(|i| {
- builder
- .append(i)
- .expect("Unable to append a value to a dictionary array.");
- });
-
- builder.finish()
- }
-}
-
-impl<T: ArrowPrimitiveType> Array for DictionaryArray<T> {
- fn as_any(&self) -> &Any {
- self
- }
-
- fn data(&self) -> &ArrayData {
- &self.data
- }
-
- fn get_buffer_memory_size(&self) -> usize {
- // Since both `keys` and `values` derive (are references from) `data`, we only need to account for `data`.
- self.data.get_buffer_memory_size()
- }
-
- fn get_array_memory_size(&self) -> usize {
- self.data.get_array_memory_size()
- + self.keys.get_array_memory_size()
- + self.values.get_array_memory_size()
- + mem::size_of_val(self)
- }
-}
-
-impl<T: ArrowPrimitiveType> fmt::Debug for DictionaryArray<T> {
- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- writeln!(
- f,
- "DictionaryArray {{keys: {:?} values: {:?}}}",
- self.keys, self.values
- )
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- use crate::{
- array::Int16Array,
- datatypes::{Int32Type, Int8Type, UInt32Type, UInt8Type},
- };
- use crate::{
- array::Int16DictionaryArray, array::PrimitiveDictionaryBuilder,
- datatypes::DataType,
- };
- use crate::{buffer::Buffer, datatypes::ToByteSlice};
-
- #[test]
- fn test_dictionary_array() {
- // Construct a value array
- let value_data = ArrayData::builder(DataType::Int8)
- .len(8)
- .add_buffer(Buffer::from(
- &[10_i8, 11, 12, 13, 14, 15, 16, 17].to_byte_slice(),
- ))
- .build();
-
- // Construct a buffer for value offsets, for the nested array:
- let keys = Buffer::from(&[2_i16, 3, 4].to_byte_slice());
-
- // Construct a dictionary array from the above two
- let key_type = DataType::Int16;
- let value_type = DataType::Int8;
- let dict_data_type =
- DataType::Dictionary(Box::new(key_type), Box::new(value_type));
- let dict_data = ArrayData::builder(dict_data_type.clone())
- .len(3)
- .add_buffer(keys.clone())
- .add_child_data(value_data.clone())
- .build();
- let dict_array = Int16DictionaryArray::from(dict_data);
-
- let values = dict_array.values();
- assert_eq!(&value_data, values.data());
- assert_eq!(DataType::Int8, dict_array.value_type());
- assert_eq!(3, dict_array.len());
-
- // Null count only makes sense in terms of the component arrays.
- assert_eq!(0, dict_array.null_count());
- assert_eq!(0, dict_array.values().null_count());
- assert_eq!(dict_array.keys(), &Int16Array::from(vec![2_i16, 3, 4]));
-
- // Now test with a non-zero offset
- let dict_data = ArrayData::builder(dict_data_type)
- .len(2)
- .offset(1)
- .add_buffer(keys)
- .add_child_data(value_data.clone())
- .build();
- let dict_array = Int16DictionaryArray::from(dict_data);
-
- let values = dict_array.values();
- assert_eq!(&value_data, values.data());
- assert_eq!(DataType::Int8, dict_array.value_type());
- assert_eq!(2, dict_array.len());
- assert_eq!(dict_array.keys(), &Int16Array::from(vec![3_i16, 4]));
- }
-
- #[test]
- fn test_dictionary_array_fmt_debug() {
- let key_builder = PrimitiveBuilder::<UInt8Type>::new(3);
- let value_builder = PrimitiveBuilder::<UInt32Type>::new(2);
- let mut builder = PrimitiveDictionaryBuilder::new(key_builder, value_builder);
- builder.append(12345678).unwrap();
- builder.append_null().unwrap();
- builder.append(22345678).unwrap();
- let array = builder.finish();
- assert_eq!(
- "DictionaryArray {keys: PrimitiveArray<UInt8>\n[\n 0,\n null,\n 1,\n] values: PrimitiveArray<UInt32>\n[\n 12345678,\n 22345678,\n]}\n",
- format!("{:?}", array)
- );
-
- let key_builder = PrimitiveBuilder::<UInt8Type>::new(20);
- let value_builder = PrimitiveBuilder::<UInt32Type>::new(2);
- let mut builder = PrimitiveDictionaryBuilder::new(key_builder, value_builder);
- for _ in 0..20 {
- builder.append(1).unwrap();
- }
- let array = builder.finish();
- assert_eq!(
- "DictionaryArray {keys: PrimitiveArray<UInt8>\n[\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n 0,\n] values: PrimitiveArray<UInt32>\n[\n 1,\n]}\n",
- format!("{:?}", array)
- );
- }
-
- #[test]
- fn test_dictionary_array_from_iter() {
- let test = vec!["a", "a", "b", "c"];
- let array: DictionaryArray<Int8Type> = test
- .iter()
- .map(|&x| if x == "b" { None } else { Some(x) })
- .collect();
- assert_eq!(
- "DictionaryArray {keys: PrimitiveArray<Int8>\n[\n 0,\n 0,\n null,\n 1,\n] values: StringArray\n[\n \"a\",\n \"c\",\n]}\n",
- format!("{:?}", array)
- );
-
- let array: DictionaryArray<Int8Type> = test.into_iter().collect();
- assert_eq!(
- "DictionaryArray {keys: PrimitiveArray<Int8>\n[\n 0,\n 0,\n 1,\n 2,\n] values: StringArray\n[\n \"a\",\n \"b\",\n \"c\",\n]}\n",
- format!("{:?}", array)
- );
- }
-
- #[test]
- fn test_dictionary_array_reverse_lookup_key() {
- let test = vec!["a", "a", "b", "c"];
- let array: DictionaryArray<Int8Type> = test.into_iter().collect();
-
- assert_eq!(array.lookup_key("c"), Some(2));
-
- // Direction of building a dictionary is the iterator direction
- let test = vec!["t3", "t3", "t2", "t2", "t1", "t3", "t4", "t1", "t0"];
- let array: DictionaryArray<Int8Type> = test.into_iter().collect();
-
- assert_eq!(array.lookup_key("t1"), Some(2));
- assert_eq!(array.lookup_key("non-existent"), None);
- }
-
- #[test]
- fn test_dictionary_keys_as_primitive_array() {
- let test = vec!["a", "b", "c", "a"];
- let array: DictionaryArray<Int8Type> = test.into_iter().collect();
-
- let keys = array.keys_array();
- assert_eq!(&DataType::Int8, keys.data_type());
- assert_eq!(0, keys.null_count());
- assert_eq!(&[0, 1, 2, 0], keys.values());
- }
-
- #[test]
- fn test_dictionary_keys_as_primitive_array_with_null() {
- let test = vec![Some("a"), None, Some("b"), None, None, Some("a")];
- let array: DictionaryArray<Int32Type> = test.into_iter().collect();
-
- let keys = array.keys_array();
- assert_eq!(&DataType::Int32, keys.data_type());
- assert_eq!(3, keys.null_count());
-
- assert_eq!(true, keys.is_valid(0));
- assert_eq!(false, keys.is_valid(1));
- assert_eq!(true, keys.is_valid(2));
- assert_eq!(false, keys.is_valid(3));
- assert_eq!(false, keys.is_valid(4));
- assert_eq!(true, keys.is_valid(5));
-
- assert_eq!(0, keys.value(0));
- assert_eq!(1, keys.value(2));
- assert_eq!(0, keys.value(5));
- }
-}
diff --git a/rust/arrow/src/array/array_list.rs b/rust/arrow/src/array/array_list.rs
deleted file mode 100644
index 0e33463..0000000
--- a/rust/arrow/src/array/array_list.rs
+++ /dev/null
@@ -1,1056 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::any::Any;
-use std::fmt;
-use std::mem;
-
-use num::Num;
-
-use super::{
- array::print_long_array, make_array, raw_pointer::RawPtrBox, Array, ArrayData,
- ArrayRef, BooleanBufferBuilder, GenericListArrayIter, PrimitiveArray,
-};
-use crate::{
- buffer::MutableBuffer,
- datatypes::{ArrowNativeType, ArrowPrimitiveType, DataType, Field},
- error::ArrowError,
-};
-
-/// trait declaring an offset size, relevant for i32 vs i64 array types.
-pub trait OffsetSizeTrait: ArrowNativeType + Num + Ord + std::ops::AddAssign {
- fn is_large() -> bool;
-}
-
-impl OffsetSizeTrait for i32 {
- #[inline]
- fn is_large() -> bool {
- false
- }
-}
-
-impl OffsetSizeTrait for i64 {
- #[inline]
- fn is_large() -> bool {
- true
- }
-}
-
-pub struct GenericListArray<OffsetSize> {
- data: ArrayData,
- values: ArrayRef,
- value_offsets: RawPtrBox<OffsetSize>,
-}
-
-impl<OffsetSize: OffsetSizeTrait> GenericListArray<OffsetSize> {
- /// Returns a reference to the values of this list.
- pub fn values(&self) -> ArrayRef {
- self.values.clone()
- }
-
- /// Returns a clone of the value type of this list.
- pub fn value_type(&self) -> DataType {
- self.values.data_ref().data_type().clone()
- }
-
- /// Returns ith value of this list array.
- /// # Safety
- /// Caller must ensure that the index is within the array bounds
- pub unsafe fn value_unchecked(&self, i: usize) -> ArrayRef {
- let end = *self.value_offsets().get_unchecked(i + 1);
- let start = *self.value_offsets().get_unchecked(i);
- self.values
- .slice(start.to_usize().unwrap(), (end - start).to_usize().unwrap())
- }
-
- /// Returns ith value of this list array.
- pub fn value(&self, i: usize) -> ArrayRef {
- let end = self.value_offsets()[i + 1];
- let start = self.value_offsets()[i];
- self.values
- .slice(start.to_usize().unwrap(), (end - start).to_usize().unwrap())
- }
-
- /// Returns the offset values in the offsets buffer
- #[inline]
- pub fn value_offsets(&self) -> &[OffsetSize] {
- // Soundness
- // pointer alignment & location is ensured by RawPtrBox
- // buffer bounds/offset is ensured by the ArrayData instance.
- unsafe {
- std::slice::from_raw_parts(
- self.value_offsets.as_ptr().add(self.data.offset()),
- self.len() + 1,
- )
- }
- }
-
- /// Returns the length for value at index `i`.
- #[inline]
- pub fn value_length(&self, i: usize) -> OffsetSize {
- let offsets = self.value_offsets();
- offsets[i + 1] - offsets[i]
- }
-
- /// constructs a new iterator
- pub fn iter<'a>(&'a self) -> GenericListArrayIter<'a, OffsetSize> {
- GenericListArrayIter::<'a, OffsetSize>::new(&self)
- }
-
- #[inline]
- fn get_type(data_type: &DataType) -> Option<&DataType> {
- if OffsetSize::is_large() {
- if let DataType::LargeList(child) = data_type {
- Some(child.data_type())
- } else {
- None
- }
- } else if let DataType::List(child) = data_type {
- Some(child.data_type())
- } else {
- None
- }
- }
-
- /// Creates a [`GenericListArray`] from an iterator of primitive values
- /// # Example
- /// ```
- /// # use arrow::array::ListArray;
- /// # use arrow::datatypes::Int32Type;
- /// let data = vec![
- /// Some(vec![Some(0), Some(1), Some(2)]),
- /// None,
- /// Some(vec![Some(3), None, Some(5)]),
- /// Some(vec![Some(6), Some(7)]),
- /// ];
- /// let list_array = ListArray::from_iter_primitive::<Int32Type, _, _>(data);
- /// println!("{:?}", list_array);
- /// ```
- pub fn from_iter_primitive<T, P, I>(iter: I) -> Self
- where
- T: ArrowPrimitiveType,
- P: AsRef<[Option<<T as ArrowPrimitiveType>::Native>]>
- + IntoIterator<Item = Option<<T as ArrowPrimitiveType>::Native>>,
- I: IntoIterator<Item = Option<P>>,
- {
- let iterator = iter.into_iter();
- let (lower, _) = iterator.size_hint();
-
- let mut offsets =
- MutableBuffer::new((lower + 1) * std::mem::size_of::<OffsetSize>());
- let mut length_so_far = OffsetSize::zero();
- offsets.push(length_so_far);
-
- let mut null_buf = BooleanBufferBuilder::new(lower);
-
- let values: PrimitiveArray<T> = iterator
- .filter_map(|maybe_slice| {
- // regardless of whether the item is Some, the offsets and null buffers must be updated.
- match &maybe_slice {
- Some(x) => {
- length_so_far +=
- OffsetSize::from_usize(x.as_ref().len()).unwrap();
- null_buf.append(true);
- }
- None => null_buf.append(false),
- };
- offsets.push(length_so_far);
- maybe_slice
- })
- .flatten()
- .collect();
-
- let field = Box::new(Field::new("item", T::DATA_TYPE, true));
- let data_type = if OffsetSize::is_large() {
- DataType::LargeList(field)
- } else {
- DataType::List(field)
- };
- let data = ArrayData::builder(data_type)
- .len(null_buf.len())
- .add_buffer(offsets.into())
- .add_child_data(values.data().clone())
- .null_bit_buffer(null_buf.into())
- .build();
- Self::from(data)
- }
-}
-
-impl<OffsetSize: OffsetSizeTrait> From<ArrayData> for GenericListArray<OffsetSize> {
- fn from(data: ArrayData) -> Self {
- Self::try_new_from_array_data(data).expect(
- "Expected infallable creation of GenericListArray from ArrayDataRef failed",
- )
- }
-}
-
-impl<OffsetSize: OffsetSizeTrait> GenericListArray<OffsetSize> {
- fn try_new_from_array_data(data: ArrayData) -> Result<Self, ArrowError> {
- if data.buffers().len() != 1 {
- return Err(ArrowError::InvalidArgumentError(
- format!("ListArray data should contain a single buffer only (value offsets), had {}",
- data.len())));
- }
-
- if data.child_data().len() != 1 {
- return Err(ArrowError::InvalidArgumentError(format!(
- "ListArray should contain a single child array (values array), had {}",
- data.child_data().len()
- )));
- }
-
- let values = data.child_data()[0].clone();
-
- if let Some(child_data_type) = Self::get_type(data.data_type()) {
- if values.data_type() != child_data_type {
- return Err(ArrowError::InvalidArgumentError(format!(
- "[Large]ListArray's child datatype {:?} does not \
- correspond to the List's datatype {:?}",
- values.data_type(),
- child_data_type
- )));
- }
- } else {
- return Err(ArrowError::InvalidArgumentError(format!(
- "[Large]ListArray's datatype must be [Large]ListArray(). It is {:?}",
- data.data_type()
- )));
- }
-
- let values = make_array(values);
- let value_offsets = data.buffers()[0].as_ptr();
-
- let value_offsets = unsafe { RawPtrBox::<OffsetSize>::new(value_offsets) };
- unsafe {
- if !(*value_offsets.as_ptr().offset(0)).is_zero() {
- return Err(ArrowError::InvalidArgumentError(String::from(
- "offsets do not start at zero",
- )));
- }
- }
- Ok(Self {
- data,
- values,
- value_offsets,
- })
- }
-}
-
-impl<OffsetSize: 'static + OffsetSizeTrait> Array for GenericListArray<OffsetSize> {
- fn as_any(&self) -> &Any {
- self
- }
-
- fn data(&self) -> &ArrayData {
- &self.data
- }
-
- /// Returns the total number of bytes of memory occupied by the buffers owned by this [ListArray].
- fn get_buffer_memory_size(&self) -> usize {
- self.data.get_buffer_memory_size()
- }
-
- /// Returns the total number of bytes of memory occupied physically by this [ListArray].
- fn get_array_memory_size(&self) -> usize {
- self.data.get_array_memory_size() + mem::size_of_val(self)
- }
-}
-
-impl<OffsetSize: OffsetSizeTrait> fmt::Debug for GenericListArray<OffsetSize> {
- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- let prefix = if OffsetSize::is_large() { "Large" } else { "" };
-
- write!(f, "{}ListArray\n[\n", prefix)?;
- print_long_array(self, f, |array, index, f| {
- fmt::Debug::fmt(&array.value(index), f)
- })?;
- write!(f, "]")
- }
-}
-
-/// A list array where each element is a variable-sized sequence of values with the same
-/// type whose memory offsets between elements are represented by a i32.
-pub type ListArray = GenericListArray<i32>;
-
-/// A list array where each element is a variable-sized sequence of values with the same
-/// type whose memory offsets between elements are represented by a i64.
-pub type LargeListArray = GenericListArray<i64>;
-
-/// A list array where each element is a fixed-size sequence of values with the same
-/// type whose maximum length is represented by a i32.
-pub struct FixedSizeListArray {
- data: ArrayData,
- values: ArrayRef,
- length: i32,
-}
-
-impl FixedSizeListArray {
- /// Returns a reference to the values of this list.
- pub fn values(&self) -> ArrayRef {
- self.values.clone()
- }
-
- /// Returns a clone of the value type of this list.
- pub fn value_type(&self) -> DataType {
- self.values.data_ref().data_type().clone()
- }
-
- /// Returns ith value of this list array.
- pub fn value(&self, i: usize) -> ArrayRef {
- self.values
- .slice(self.value_offset(i) as usize, self.value_length() as usize)
- }
-
- /// Returns the offset for value at index `i`.
- ///
- /// Note this doesn't do any bound checking, for performance reason.
- #[inline]
- pub fn value_offset(&self, i: usize) -> i32 {
- self.value_offset_at(self.data.offset() + i)
- }
-
- /// Returns the length for value at index `i`.
- ///
- /// Note this doesn't do any bound checking, for performance reason.
- #[inline]
- pub const fn value_length(&self) -> i32 {
- self.length
- }
-
- #[inline]
- const fn value_offset_at(&self, i: usize) -> i32 {
- i as i32 * self.length
- }
-}
-
-impl From<ArrayData> for FixedSizeListArray {
- fn from(data: ArrayData) -> Self {
- assert_eq!(
- data.buffers().len(),
- 0,
- "FixedSizeListArray data should not contain a buffer for value offsets"
- );
- assert_eq!(
- data.child_data().len(),
- 1,
- "FixedSizeListArray should contain a single child array (values array)"
- );
- let values = make_array(data.child_data()[0].clone());
- let length = match data.data_type() {
- DataType::FixedSizeList(_, len) => {
- if *len > 0 {
- // check that child data is multiple of length
- assert_eq!(
- values.len() % *len as usize,
- 0,
- "FixedSizeListArray child array length should be a multiple of {}",
- len
- );
- }
-
- *len
- }
- _ => {
- panic!("FixedSizeListArray data should contain a FixedSizeList data type")
- }
- };
- Self {
- data,
- values,
- length,
- }
- }
-}
-
-impl Array for FixedSizeListArray {
- fn as_any(&self) -> &Any {
- self
- }
-
- fn data(&self) -> &ArrayData {
- &self.data
- }
-
- /// Returns the total number of bytes of memory occupied by the buffers owned by this [FixedSizeListArray].
- fn get_buffer_memory_size(&self) -> usize {
- self.data.get_buffer_memory_size() + self.values().get_buffer_memory_size()
- }
-
- /// Returns the total number of bytes of memory occupied physically by this [FixedSizeListArray].
- fn get_array_memory_size(&self) -> usize {
- self.data.get_array_memory_size()
- + self.values().get_array_memory_size()
- + mem::size_of_val(self)
- }
-}
-
-impl fmt::Debug for FixedSizeListArray {
- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- write!(f, "FixedSizeListArray<{}>\n[\n", self.value_length())?;
- print_long_array(self, f, |array, index, f| {
- fmt::Debug::fmt(&array.value(index), f)
- })?;
- write!(f, "]")
- }
-}
-
-#[cfg(test)]
-mod tests {
- use crate::{
- alloc,
- array::ArrayData,
- array::Int32Array,
- buffer::Buffer,
- datatypes::Field,
- datatypes::{Int32Type, ToByteSlice},
- util::bit_util,
- };
-
- use super::*;
-
- fn create_from_buffers() -> ListArray {
- // Construct a value array
- let value_data = ArrayData::builder(DataType::Int32)
- .len(8)
- .add_buffer(Buffer::from(&[0, 1, 2, 3, 4, 5, 6, 7].to_byte_slice()))
- .build();
-
- // Construct a buffer for value offsets, for the nested array:
- // [[0, 1, 2], [3, 4, 5], [6, 7]]
- let value_offsets = Buffer::from(&[0, 3, 6, 8].to_byte_slice());
-
- // Construct a list array from the above two
- let list_data_type =
- DataType::List(Box::new(Field::new("item", DataType::Int32, true)));
- let list_data = ArrayData::builder(list_data_type)
- .len(3)
- .add_buffer(value_offsets)
- .add_child_data(value_data)
- .build();
- ListArray::from(list_data)
- }
-
- #[test]
- fn test_from_iter_primitive() {
- let data = vec![
- Some(vec![Some(0), Some(1), Some(2)]),
- Some(vec![Some(3), Some(4), Some(5)]),
- Some(vec![Some(6), Some(7)]),
- ];
- let list_array = ListArray::from_iter_primitive::<Int32Type, _, _>(data);
-
- let another = create_from_buffers();
- assert_eq!(list_array, another)
- }
-
- #[test]
- fn test_list_array() {
- // Construct a value array
- let value_data = ArrayData::builder(DataType::Int32)
- .len(8)
- .add_buffer(Buffer::from_slice_ref(&[0, 1, 2, 3, 4, 5, 6, 7]))
- .build();
-
- // Construct a buffer for value offsets, for the nested array:
- // [[0, 1, 2], [3, 4, 5], [6, 7]]
- let value_offsets = Buffer::from_slice_ref(&[0, 3, 6, 8]);
-
- // Construct a list array from the above two
- let list_data_type =
- DataType::List(Box::new(Field::new("item", DataType::Int32, false)));
- let list_data = ArrayData::builder(list_data_type.clone())
- .len(3)
- .add_buffer(value_offsets.clone())
- .add_child_data(value_data.clone())
- .build();
- let list_array = ListArray::from(list_data);
-
- let values = list_array.values();
- assert_eq!(&value_data, values.data());
- assert_eq!(DataType::Int32, list_array.value_type());
- assert_eq!(3, list_array.len());
- assert_eq!(0, list_array.null_count());
- assert_eq!(6, list_array.value_offsets()[2]);
- assert_eq!(2, list_array.value_length(2));
- assert_eq!(
- 0,
- list_array
- .value(0)
- .as_any()
- .downcast_ref::<Int32Array>()
- .unwrap()
- .value(0)
- );
- assert_eq!(
- 0,
- unsafe { list_array.value_unchecked(0) }
- .as_any()
- .downcast_ref::<Int32Array>()
- .unwrap()
- .value(0)
- );
- for i in 0..3 {
- assert!(list_array.is_valid(i));
- assert!(!list_array.is_null(i));
- }
-
- // Now test with a non-zero offset
- let list_data = ArrayData::builder(list_data_type)
- .len(3)
- .offset(1)
- .add_buffer(value_offsets)
- .add_child_data(value_data.clone())
- .build();
- let list_array = ListArray::from(list_data);
-
- let values = list_array.values();
- assert_eq!(&value_data, values.data());
- assert_eq!(DataType::Int32, list_array.value_type());
- assert_eq!(3, list_array.len());
- assert_eq!(0, list_array.null_count());
- assert_eq!(6, list_array.value_offsets()[1]);
- assert_eq!(2, list_array.value_length(1));
- assert_eq!(
- 3,
- list_array
- .value(0)
- .as_any()
- .downcast_ref::<Int32Array>()
- .unwrap()
- .value(0)
- );
- assert_eq!(
- 3,
- unsafe { list_array.value_unchecked(0) }
- .as_any()
- .downcast_ref::<Int32Array>()
- .unwrap()
- .value(0)
- );
- }
-
- #[test]
- fn test_large_list_array() {
- // Construct a value array
- let value_data = ArrayData::builder(DataType::Int32)
- .len(8)
- .add_buffer(Buffer::from_slice_ref(&[0, 1, 2, 3, 4, 5, 6, 7]))
- .build();
-
- // Construct a buffer for value offsets, for the nested array:
- // [[0, 1, 2], [3, 4, 5], [6, 7]]
- let value_offsets = Buffer::from_slice_ref(&[0i64, 3, 6, 8]);
-
- // Construct a list array from the above two
- let list_data_type =
- DataType::LargeList(Box::new(Field::new("item", DataType::Int32, false)));
- let list_data = ArrayData::builder(list_data_type.clone())
- .len(3)
- .add_buffer(value_offsets.clone())
- .add_child_data(value_data.clone())
- .build();
- let list_array = LargeListArray::from(list_data);
-
- let values = list_array.values();
- assert_eq!(&value_data, values.data());
- assert_eq!(DataType::Int32, list_array.value_type());
- assert_eq!(3, list_array.len());
- assert_eq!(0, list_array.null_count());
- assert_eq!(6, list_array.value_offsets()[2]);
- assert_eq!(2, list_array.value_length(2));
- assert_eq!(
- 0,
- list_array
- .value(0)
- .as_any()
- .downcast_ref::<Int32Array>()
- .unwrap()
- .value(0)
- );
- assert_eq!(
- 0,
- unsafe { list_array.value_unchecked(0) }
- .as_any()
- .downcast_ref::<Int32Array>()
- .unwrap()
- .value(0)
- );
- for i in 0..3 {
- assert!(list_array.is_valid(i));
- assert!(!list_array.is_null(i));
- }
-
- // Now test with a non-zero offset
- let list_data = ArrayData::builder(list_data_type)
- .len(3)
- .offset(1)
- .add_buffer(value_offsets)
- .add_child_data(value_data.clone())
- .build();
- let list_array = LargeListArray::from(list_data);
-
- let values = list_array.values();
- assert_eq!(&value_data, values.data());
- assert_eq!(DataType::Int32, list_array.value_type());
- assert_eq!(3, list_array.len());
- assert_eq!(0, list_array.null_count());
- assert_eq!(6, list_array.value_offsets()[1]);
- assert_eq!(2, list_array.value_length(1));
- assert_eq!(
- 3,
- list_array
- .value(0)
- .as_any()
- .downcast_ref::<Int32Array>()
- .unwrap()
- .value(0)
- );
- assert_eq!(
- 3,
- unsafe { list_array.value_unchecked(0) }
- .as_any()
- .downcast_ref::<Int32Array>()
- .unwrap()
- .value(0)
- );
- }
-
- #[test]
- fn test_fixed_size_list_array() {
- // Construct a value array
- let value_data = ArrayData::builder(DataType::Int32)
- .len(9)
- .add_buffer(Buffer::from_slice_ref(&[0, 1, 2, 3, 4, 5, 6, 7, 8]))
- .build();
-
- // Construct a list array from the above two
- let list_data_type = DataType::FixedSizeList(
- Box::new(Field::new("item", DataType::Int32, false)),
- 3,
- );
- let list_data = ArrayData::builder(list_data_type.clone())
- .len(3)
- .add_child_data(value_data.clone())
- .build();
- let list_array = FixedSizeListArray::from(list_data);
-
- let values = list_array.values();
- assert_eq!(&value_data, values.data());
- assert_eq!(DataType::Int32, list_array.value_type());
- assert_eq!(3, list_array.len());
- assert_eq!(0, list_array.null_count());
- assert_eq!(6, list_array.value_offset(2));
- assert_eq!(3, list_array.value_length());
- assert_eq!(
- 0,
- list_array
- .value(0)
- .as_any()
- .downcast_ref::<Int32Array>()
- .unwrap()
- .value(0)
- );
- for i in 0..3 {
- assert!(list_array.is_valid(i));
- assert!(!list_array.is_null(i));
- }
-
- // Now test with a non-zero offset
- let list_data = ArrayData::builder(list_data_type)
- .len(3)
- .offset(1)
- .add_child_data(value_data.clone())
- .build();
- let list_array = FixedSizeListArray::from(list_data);
-
- let values = list_array.values();
- assert_eq!(&value_data, values.data());
- assert_eq!(DataType::Int32, list_array.value_type());
- assert_eq!(3, list_array.len());
- assert_eq!(0, list_array.null_count());
- assert_eq!(
- 3,
- list_array
- .value(0)
- .as_any()
- .downcast_ref::<Int32Array>()
- .unwrap()
- .value(0)
- );
- assert_eq!(6, list_array.value_offset(1));
- assert_eq!(3, list_array.value_length());
- }
-
- #[test]
- #[should_panic(
- expected = "FixedSizeListArray child array length should be a multiple of 3"
- )]
- fn test_fixed_size_list_array_unequal_children() {
- // Construct a value array
- let value_data = ArrayData::builder(DataType::Int32)
- .len(8)
- .add_buffer(Buffer::from_slice_ref(&[0, 1, 2, 3, 4, 5, 6, 7]))
- .build();
-
- // Construct a list array from the above two
- let list_data_type = DataType::FixedSizeList(
- Box::new(Field::new("item", DataType::Int32, false)),
- 3,
- );
- let list_data = ArrayData::builder(list_data_type)
- .len(3)
- .add_child_data(value_data)
- .build();
- FixedSizeListArray::from(list_data);
- }
-
- #[test]
- fn test_list_array_slice() {
- // Construct a value array
- let value_data = ArrayData::builder(DataType::Int32)
- .len(10)
- .add_buffer(Buffer::from_slice_ref(&[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]))
- .build();
-
- // Construct a buffer for value offsets, for the nested array:
- // [[0, 1], null, null, [2, 3], [4, 5], null, [6, 7, 8], null, [9]]
- let value_offsets = Buffer::from_slice_ref(&[0, 2, 2, 2, 4, 6, 6, 9, 9, 10]);
- // 01011001 00000001
- let mut null_bits: [u8; 2] = [0; 2];
- bit_util::set_bit(&mut null_bits, 0);
- bit_util::set_bit(&mut null_bits, 3);
- bit_util::set_bit(&mut null_bits, 4);
- bit_util::set_bit(&mut null_bits, 6);
- bit_util::set_bit(&mut null_bits, 8);
-
- // Construct a list array from the above two
- let list_data_type =
- DataType::List(Box::new(Field::new("item", DataType::Int32, false)));
- let list_data = ArrayData::builder(list_data_type)
- .len(9)
- .add_buffer(value_offsets)
- .add_child_data(value_data.clone())
- .null_bit_buffer(Buffer::from(null_bits))
- .build();
- let list_array = ListArray::from(list_data);
-
- let values = list_array.values();
- assert_eq!(&value_data, values.data());
- assert_eq!(DataType::Int32, list_array.value_type());
- assert_eq!(9, list_array.len());
- assert_eq!(4, list_array.null_count());
- assert_eq!(2, list_array.value_offsets()[3]);
- assert_eq!(2, list_array.value_length(3));
-
- let sliced_array = list_array.slice(1, 6);
- assert_eq!(6, sliced_array.len());
- assert_eq!(1, sliced_array.offset());
- assert_eq!(3, sliced_array.null_count());
-
- for i in 0..sliced_array.len() {
- if bit_util::get_bit(&null_bits, sliced_array.offset() + i) {
- assert!(sliced_array.is_valid(i));
- } else {
- assert!(sliced_array.is_null(i));
- }
- }
-
- // Check offset and length for each non-null value.
- let sliced_list_array =
- sliced_array.as_any().downcast_ref::<ListArray>().unwrap();
- assert_eq!(2, sliced_list_array.value_offsets()[2]);
- assert_eq!(2, sliced_list_array.value_length(2));
- assert_eq!(4, sliced_list_array.value_offsets()[3]);
- assert_eq!(2, sliced_list_array.value_length(3));
- assert_eq!(6, sliced_list_array.value_offsets()[5]);
- assert_eq!(3, sliced_list_array.value_length(5));
- }
-
- #[test]
- fn test_large_list_array_slice() {
- // Construct a value array
- let value_data = ArrayData::builder(DataType::Int32)
- .len(10)
- .add_buffer(Buffer::from_slice_ref(&[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]))
- .build();
-
- // Construct a buffer for value offsets, for the nested array:
- // [[0, 1], null, null, [2, 3], [4, 5], null, [6, 7, 8], null, [9]]
- let value_offsets = Buffer::from_slice_ref(&[0i64, 2, 2, 2, 4, 6, 6, 9, 9, 10]);
- // 01011001 00000001
- let mut null_bits: [u8; 2] = [0; 2];
- bit_util::set_bit(&mut null_bits, 0);
- bit_util::set_bit(&mut null_bits, 3);
- bit_util::set_bit(&mut null_bits, 4);
- bit_util::set_bit(&mut null_bits, 6);
- bit_util::set_bit(&mut null_bits, 8);
-
- // Construct a list array from the above two
- let list_data_type =
- DataType::LargeList(Box::new(Field::new("item", DataType::Int32, false)));
- let list_data = ArrayData::builder(list_data_type)
- .len(9)
- .add_buffer(value_offsets)
- .add_child_data(value_data.clone())
- .null_bit_buffer(Buffer::from(null_bits))
- .build();
- let list_array = LargeListArray::from(list_data);
-
- let values = list_array.values();
- assert_eq!(&value_data, values.data());
- assert_eq!(DataType::Int32, list_array.value_type());
- assert_eq!(9, list_array.len());
- assert_eq!(4, list_array.null_count());
- assert_eq!(2, list_array.value_offsets()[3]);
- assert_eq!(2, list_array.value_length(3));
-
- let sliced_array = list_array.slice(1, 6);
- assert_eq!(6, sliced_array.len());
- assert_eq!(1, sliced_array.offset());
- assert_eq!(3, sliced_array.null_count());
-
- for i in 0..sliced_array.len() {
- if bit_util::get_bit(&null_bits, sliced_array.offset() + i) {
- assert!(sliced_array.is_valid(i));
- } else {
- assert!(sliced_array.is_null(i));
- }
- }
-
- // Check offset and length for each non-null value.
- let sliced_list_array = sliced_array
- .as_any()
- .downcast_ref::<LargeListArray>()
- .unwrap();
- assert_eq!(2, sliced_list_array.value_offsets()[2]);
- assert_eq!(2, sliced_list_array.value_length(2));
- assert_eq!(4, sliced_list_array.value_offsets()[3]);
- assert_eq!(2, sliced_list_array.value_length(3));
- assert_eq!(6, sliced_list_array.value_offsets()[5]);
- assert_eq!(3, sliced_list_array.value_length(5));
- }
-
- #[test]
- #[should_panic(expected = "index out of bounds: the len is 10 but the index is 11")]
- fn test_list_array_index_out_of_bound() {
- // Construct a value array
- let value_data = ArrayData::builder(DataType::Int32)
- .len(10)
- .add_buffer(Buffer::from_slice_ref(&[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]))
- .build();
-
- // Construct a buffer for value offsets, for the nested array:
- // [[0, 1], null, null, [2, 3], [4, 5], null, [6, 7, 8], null, [9]]
- let value_offsets = Buffer::from_slice_ref(&[0i64, 2, 2, 2, 4, 6, 6, 9, 9, 10]);
- // 01011001 00000001
- let mut null_bits: [u8; 2] = [0; 2];
- bit_util::set_bit(&mut null_bits, 0);
- bit_util::set_bit(&mut null_bits, 3);
- bit_util::set_bit(&mut null_bits, 4);
- bit_util::set_bit(&mut null_bits, 6);
- bit_util::set_bit(&mut null_bits, 8);
-
- // Construct a list array from the above two
- let list_data_type =
- DataType::LargeList(Box::new(Field::new("item", DataType::Int32, false)));
- let list_data = ArrayData::builder(list_data_type)
- .len(9)
- .add_buffer(value_offsets)
- .add_child_data(value_data)
- .null_bit_buffer(Buffer::from(null_bits))
- .build();
- let list_array = LargeListArray::from(list_data);
- assert_eq!(9, list_array.len());
-
- list_array.value(10);
- }
-
- #[test]
- fn test_fixed_size_list_array_slice() {
- // Construct a value array
- let value_data = ArrayData::builder(DataType::Int32)
- .len(10)
- .add_buffer(Buffer::from_slice_ref(&[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]))
- .build();
-
- // Set null buts for the nested array:
- // [[0, 1], null, null, [6, 7], [8, 9]]
- // 01011001 00000001
- let mut null_bits: [u8; 1] = [0; 1];
- bit_util::set_bit(&mut null_bits, 0);
- bit_util::set_bit(&mut null_bits, 3);
- bit_util::set_bit(&mut null_bits, 4);
-
- // Construct a fixed size list array from the above two
- let list_data_type = DataType::FixedSizeList(
- Box::new(Field::new("item", DataType::Int32, false)),
- 2,
- );
- let list_data = ArrayData::builder(list_data_type)
- .len(5)
- .add_child_data(value_data.clone())
- .null_bit_buffer(Buffer::from(null_bits))
- .build();
- let list_array = FixedSizeListArray::from(list_data);
-
- let values = list_array.values();
- assert_eq!(&value_data, values.data());
- assert_eq!(DataType::Int32, list_array.value_type());
- assert_eq!(5, list_array.len());
- assert_eq!(2, list_array.null_count());
- assert_eq!(6, list_array.value_offset(3));
- assert_eq!(2, list_array.value_length());
-
- let sliced_array = list_array.slice(1, 4);
- assert_eq!(4, sliced_array.len());
- assert_eq!(1, sliced_array.offset());
- assert_eq!(2, sliced_array.null_count());
-
- for i in 0..sliced_array.len() {
- if bit_util::get_bit(&null_bits, sliced_array.offset() + i) {
- assert!(sliced_array.is_valid(i));
- } else {
- assert!(sliced_array.is_null(i));
- }
- }
-
- // Check offset and length for each non-null value.
- let sliced_list_array = sliced_array
- .as_any()
- .downcast_ref::<FixedSizeListArray>()
- .unwrap();
- assert_eq!(2, sliced_list_array.value_length());
- assert_eq!(6, sliced_list_array.value_offset(2));
- assert_eq!(8, sliced_list_array.value_offset(3));
- }
-
- #[test]
- #[should_panic(expected = "assertion failed: (offset + length) <= self.len()")]
- fn test_fixed_size_list_array_index_out_of_bound() {
- // Construct a value array
- let value_data = ArrayData::builder(DataType::Int32)
- .len(10)
- .add_buffer(Buffer::from_slice_ref(&[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]))
- .build();
-
- // Set null buts for the nested array:
- // [[0, 1], null, null, [6, 7], [8, 9]]
- // 01011001 00000001
- let mut null_bits: [u8; 1] = [0; 1];
- bit_util::set_bit(&mut null_bits, 0);
- bit_util::set_bit(&mut null_bits, 3);
- bit_util::set_bit(&mut null_bits, 4);
-
- // Construct a fixed size list array from the above two
- let list_data_type = DataType::FixedSizeList(
- Box::new(Field::new("item", DataType::Int32, false)),
- 2,
- );
- let list_data = ArrayData::builder(list_data_type)
- .len(5)
- .add_child_data(value_data)
- .null_bit_buffer(Buffer::from(null_bits))
- .build();
- let list_array = FixedSizeListArray::from(list_data);
-
- list_array.value(10);
- }
-
- #[test]
- #[should_panic(
- expected = "ListArray data should contain a single buffer only (value offsets)"
- )]
- fn test_list_array_invalid_buffer_len() {
- let value_data = ArrayData::builder(DataType::Int32)
- .len(8)
- .add_buffer(Buffer::from_slice_ref(&[0, 1, 2, 3, 4, 5, 6, 7]))
- .build();
- let list_data_type =
- DataType::List(Box::new(Field::new("item", DataType::Int32, false)));
- let list_data = ArrayData::builder(list_data_type)
- .len(3)
- .add_child_data(value_data)
- .build();
- ListArray::from(list_data);
- }
-
- #[test]
- #[should_panic(
- expected = "ListArray should contain a single child array (values array)"
- )]
- fn test_list_array_invalid_child_array_len() {
- let value_offsets = Buffer::from_slice_ref(&[0, 2, 5, 7]);
- let list_data_type =
- DataType::List(Box::new(Field::new("item", DataType::Int32, false)));
- let list_data = ArrayData::builder(list_data_type)
- .len(3)
- .add_buffer(value_offsets)
- .build();
- ListArray::from(list_data);
- }
-
- #[test]
- #[should_panic(expected = "offsets do not start at zero")]
- fn test_list_array_invalid_value_offset_start() {
- let value_data = ArrayData::builder(DataType::Int32)
- .len(8)
- .add_buffer(Buffer::from_slice_ref(&[0, 1, 2, 3, 4, 5, 6, 7]))
- .build();
-
- let value_offsets = Buffer::from_slice_ref(&[2, 2, 5, 7]);
-
- let list_data_type =
- DataType::List(Box::new(Field::new("item", DataType::Int32, false)));
- let list_data = ArrayData::builder(list_data_type)
- .len(3)
- .add_buffer(value_offsets)
- .add_child_data(value_data)
- .build();
- ListArray::from(list_data);
- }
-
- #[test]
- #[should_panic(expected = "memory is not aligned")]
- fn test_primitive_array_alignment() {
- let ptr = alloc::allocate_aligned::<u8>(8);
- let buf = unsafe { Buffer::from_raw_parts(ptr, 8, 8) };
- let buf2 = buf.slice(1);
- let array_data = ArrayData::builder(DataType::Int32).add_buffer(buf2).build();
- Int32Array::from(array_data);
- }
-
- #[test]
- #[should_panic(expected = "memory is not aligned")]
- fn test_list_array_alignment() {
- let ptr = alloc::allocate_aligned::<u8>(8);
- let buf = unsafe { Buffer::from_raw_parts(ptr, 8, 8) };
- let buf2 = buf.slice(1);
-
- let values: [i32; 8] = [0; 8];
- let value_data = ArrayData::builder(DataType::Int32)
- .add_buffer(Buffer::from_slice_ref(&values))
- .build();
-
- let list_data_type =
- DataType::List(Box::new(Field::new("item", DataType::Int32, false)));
- let list_data = ArrayData::builder(list_data_type)
- .add_buffer(buf2)
- .add_child_data(value_data)
- .build();
- ListArray::from(list_data);
- }
-}
diff --git a/rust/arrow/src/array/array_primitive.rs b/rust/arrow/src/array/array_primitive.rs
deleted file mode 100644
index d2b3b66..0000000
--- a/rust/arrow/src/array/array_primitive.rs
+++ /dev/null
@@ -1,942 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::any::Any;
-use std::borrow::Borrow;
-use std::convert::From;
-use std::fmt;
-use std::iter::{FromIterator, IntoIterator};
-use std::mem;
-
-use chrono::{prelude::*, Duration};
-
-use super::array::print_long_array;
-use super::raw_pointer::RawPtrBox;
-use super::*;
-use crate::temporal_conversions;
-use crate::util::bit_util;
-use crate::{
- buffer::{Buffer, MutableBuffer},
- util::trusted_len_unzip,
-};
-
-/// Number of seconds in a day
-const SECONDS_IN_DAY: i64 = 86_400;
-/// Number of milliseconds in a second
-const MILLISECONDS: i64 = 1_000;
-/// Number of microseconds in a second
-const MICROSECONDS: i64 = 1_000_000;
-/// Number of nanoseconds in a second
-const NANOSECONDS: i64 = 1_000_000_000;
-
-/// Array whose elements are of primitive types.
-pub struct PrimitiveArray<T: ArrowPrimitiveType> {
- /// Underlying ArrayData
- /// # Safety
- /// must have exactly one buffer, aligned to type T
- data: ArrayData,
- /// Pointer to the value array. The lifetime of this must be <= to the value buffer
- /// stored in `data`, so it's safe to store.
- /// # Safety
- /// raw_values must have a value equivalent to `data.buffers()[0].raw_data()`
- /// raw_values must have alignment for type T::NativeType
- raw_values: RawPtrBox<T::Native>,
-}
-
-impl<T: ArrowPrimitiveType> PrimitiveArray<T> {
- /// Returns the length of this array.
- #[inline]
- pub fn len(&self) -> usize {
- self.data.len()
- }
-
- /// Returns whether this array is empty.
- pub fn is_empty(&self) -> bool {
- self.data.is_empty()
- }
-
- /// Returns a slice of the values of this array
- #[inline]
- pub fn values(&self) -> &[T::Native] {
- // Soundness
- // raw_values alignment & location is ensured by fn from(ArrayDataRef)
- // buffer bounds/offset is ensured by the ArrayData instance.
- unsafe {
- std::slice::from_raw_parts(
- self.raw_values.as_ptr().add(self.data.offset()),
- self.len(),
- )
- }
- }
-
- // Returns a new primitive array builder
- pub fn builder(capacity: usize) -> PrimitiveBuilder<T> {
- PrimitiveBuilder::<T>::new(capacity)
- }
-
- /// Returns the primitive value at index `i`.
- ///
- /// # Safety
- ///
- /// caller must ensure that the passed in offset is less than the array len()
- pub unsafe fn value_unchecked(&self, i: usize) -> T::Native {
- let offset = i + self.offset();
- *self.raw_values.as_ptr().add(offset)
- }
-
- /// Returns the primitive value at index `i`.
- ///
- /// Note this doesn't do any bound checking, for performance reason.
- /// # Safety
- /// caller must ensure that the passed in offset is less than the array len()
- pub fn value(&self, i: usize) -> T::Native {
- debug_assert!(i < self.len());
- unsafe { self.value_unchecked(i) }
- }
-
- /// Creates a PrimitiveArray based on an iterator of values without nulls
- pub fn from_iter_values<I: IntoIterator<Item = T::Native>>(iter: I) -> Self {
- let val_buf: Buffer = iter.into_iter().collect();
- let data = ArrayData::new(
- T::DATA_TYPE,
- val_buf.len() / mem::size_of::<<T as ArrowPrimitiveType>::Native>(),
- None,
- None,
- 0,
- vec![val_buf],
- vec![],
- );
- PrimitiveArray::from(data)
- }
-
- /// Creates a PrimitiveArray based on a constant value with `count` elements
- pub fn from_value(value: T::Native, count: usize) -> Self {
- // # Safety: length is known
- let val_buf = unsafe { Buffer::from_trusted_len_iter((0..count).map(|_| value)) };
- let data = ArrayData::new(
- T::DATA_TYPE,
- val_buf.len() / mem::size_of::<<T as ArrowPrimitiveType>::Native>(),
- None,
- None,
- 0,
- vec![val_buf],
- vec![],
- );
- PrimitiveArray::from(data)
- }
-}
-
-impl<T: ArrowPrimitiveType> Array for PrimitiveArray<T> {
- fn as_any(&self) -> &Any {
- self
- }
-
- fn data(&self) -> &ArrayData {
- &self.data
- }
-
- /// Returns the total number of bytes of memory occupied by the buffers owned by this [PrimitiveArray].
- fn get_buffer_memory_size(&self) -> usize {
- self.data.get_buffer_memory_size()
- }
-
- /// Returns the total number of bytes of memory occupied physically by this [PrimitiveArray].
- fn get_array_memory_size(&self) -> usize {
- self.data.get_array_memory_size() + mem::size_of::<RawPtrBox<T::Native>>()
- }
-}
-
-fn as_datetime<T: ArrowPrimitiveType>(v: i64) -> Option<NaiveDateTime> {
- match T::DATA_TYPE {
- DataType::Date32 => Some(temporal_conversions::date32_to_datetime(v as i32)),
- DataType::Date64 => Some(temporal_conversions::date64_to_datetime(v)),
- DataType::Time32(_) | DataType::Time64(_) => None,
- DataType::Timestamp(unit, _) => match unit {
- TimeUnit::Second => Some(temporal_conversions::timestamp_s_to_datetime(v)),
- TimeUnit::Millisecond => {
- Some(temporal_conversions::timestamp_ms_to_datetime(v))
- }
- TimeUnit::Microsecond => {
- Some(temporal_conversions::timestamp_us_to_datetime(v))
- }
- TimeUnit::Nanosecond => {
- Some(temporal_conversions::timestamp_ns_to_datetime(v))
- }
- },
- // interval is not yet fully documented [ARROW-3097]
- DataType::Interval(_) => None,
- _ => None,
- }
-}
-
-fn as_date<T: ArrowPrimitiveType>(v: i64) -> Option<NaiveDate> {
- as_datetime::<T>(v).map(|datetime| datetime.date())
-}
-
-fn as_time<T: ArrowPrimitiveType>(v: i64) -> Option<NaiveTime> {
- match T::DATA_TYPE {
- DataType::Time32(unit) => {
- // safe to immediately cast to u32 as `self.value(i)` is positive i32
- let v = v as u32;
- match unit {
- TimeUnit::Second => Some(temporal_conversions::time32s_to_time(v as i32)),
- TimeUnit::Millisecond => {
- Some(temporal_conversions::time32ms_to_time(v as i32))
- }
- _ => None,
- }
- }
- DataType::Time64(unit) => match unit {
- TimeUnit::Microsecond => Some(temporal_conversions::time64us_to_time(v)),
- TimeUnit::Nanosecond => Some(temporal_conversions::time64ns_to_time(v)),
- _ => None,
- },
- DataType::Timestamp(_, _) => as_datetime::<T>(v).map(|datetime| datetime.time()),
- DataType::Date32 | DataType::Date64 => Some(NaiveTime::from_hms(0, 0, 0)),
- DataType::Interval(_) => None,
- _ => None,
- }
-}
-
-fn as_duration<T: ArrowPrimitiveType>(v: i64) -> Option<Duration> {
- match T::DATA_TYPE {
- DataType::Duration(unit) => match unit {
- TimeUnit::Second => Some(temporal_conversions::duration_s_to_duration(v)),
- TimeUnit::Millisecond => {
- Some(temporal_conversions::duration_ms_to_duration(v))
- }
- TimeUnit::Microsecond => {
- Some(temporal_conversions::duration_us_to_duration(v))
- }
- TimeUnit::Nanosecond => {
- Some(temporal_conversions::duration_ns_to_duration(v))
- }
- },
- _ => None,
- }
-}
-
-impl<T: ArrowTemporalType + ArrowNumericType> PrimitiveArray<T>
-where
- i64: std::convert::From<T::Native>,
-{
- /// Returns value as a chrono `NaiveDateTime`, handling time resolution
- ///
- /// If a data type cannot be converted to `NaiveDateTime`, a `None` is returned.
- /// A valid value is expected, thus the user should first check for validity.
- pub fn value_as_datetime(&self, i: usize) -> Option<NaiveDateTime> {
- as_datetime::<T>(i64::from(self.value(i)))
- }
-
- /// Returns value as a chrono `NaiveDate` by using `Self::datetime()`
- ///
- /// If a data type cannot be converted to `NaiveDate`, a `None` is returned
- pub fn value_as_date(&self, i: usize) -> Option<NaiveDate> {
- self.value_as_datetime(i).map(|datetime| datetime.date())
- }
-
- /// Returns a value as a chrono `NaiveTime`
- ///
- /// `Date32` and `Date64` return UTC midnight as they do not have time resolution
- pub fn value_as_time(&self, i: usize) -> Option<NaiveTime> {
- as_time::<T>(i64::from(self.value(i)))
- }
-
- /// Returns a value as a chrono `Duration`
- ///
- /// If a data type cannot be converted to `Duration`, a `None` is returned
- pub fn value_as_duration(&self, i: usize) -> Option<Duration> {
- as_duration::<T>(i64::from(self.value(i)))
- }
-}
-
-impl<T: ArrowPrimitiveType> fmt::Debug for PrimitiveArray<T> {
- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- write!(f, "PrimitiveArray<{:?}>\n[\n", T::DATA_TYPE)?;
- print_long_array(self, f, |array, index, f| match T::DATA_TYPE {
- DataType::Date32 | DataType::Date64 => {
- let v = self.value(index).to_isize().unwrap() as i64;
- match as_date::<T>(v) {
- Some(date) => write!(f, "{:?}", date),
- None => write!(f, "null"),
- }
- }
- DataType::Time32(_) | DataType::Time64(_) => {
- let v = self.value(index).to_isize().unwrap() as i64;
- match as_time::<T>(v) {
- Some(time) => write!(f, "{:?}", time),
- None => write!(f, "null"),
- }
- }
- DataType::Timestamp(_, _) => {
- let v = self.value(index).to_isize().unwrap() as i64;
- match as_datetime::<T>(v) {
- Some(datetime) => write!(f, "{:?}", datetime),
- None => write!(f, "null"),
- }
- }
- _ => fmt::Debug::fmt(&array.value(index), f),
- })?;
- write!(f, "]")
- }
-}
-
-impl<'a, T: ArrowPrimitiveType> IntoIterator for &'a PrimitiveArray<T> {
- type Item = Option<<T as ArrowPrimitiveType>::Native>;
- type IntoIter = PrimitiveIter<'a, T>;
-
- fn into_iter(self) -> Self::IntoIter {
- PrimitiveIter::<'a, T>::new(self)
- }
-}
-
-impl<'a, T: ArrowPrimitiveType> PrimitiveArray<T> {
- /// constructs a new iterator
- pub fn iter(&'a self) -> PrimitiveIter<'a, T> {
- PrimitiveIter::<'a, T>::new(&self)
- }
-}
-
-impl<T: ArrowPrimitiveType, Ptr: Borrow<Option<<T as ArrowPrimitiveType>::Native>>>
- FromIterator<Ptr> for PrimitiveArray<T>
-{
- fn from_iter<I: IntoIterator<Item = Ptr>>(iter: I) -> Self {
- let iter = iter.into_iter();
- let (lower, _) = iter.size_hint();
-
- let mut null_buf = BooleanBufferBuilder::new(lower);
-
- let buffer: Buffer = iter
- .map(|item| {
- if let Some(a) = item.borrow() {
- null_buf.append(true);
- *a
- } else {
- null_buf.append(false);
- // this ensures that null items on the buffer are not arbitrary.
- // This is important because falible operations can use null values (e.g. a vectorized "add")
- // which may panic (e.g. overflow if the number on the slots happen to be very large).
- T::Native::default()
- }
- })
- .collect();
-
- let data = ArrayData::new(
- T::DATA_TYPE,
- null_buf.len(),
- None,
- Some(null_buf.into()),
- 0,
- vec![buffer],
- vec![],
- );
- PrimitiveArray::from(data)
- }
-}
-
-impl<T: ArrowPrimitiveType> PrimitiveArray<T> {
- /// Creates a [`PrimitiveArray`] from an iterator of trusted length.
- /// # Safety
- /// The iterator must be [`TrustedLen`](https://doc.rust-lang.org/std/iter/trait.TrustedLen.html).
- /// I.e. that `size_hint().1` correctly reports its length.
- #[inline]
- pub unsafe fn from_trusted_len_iter<I, P>(iter: I) -> Self
- where
- P: std::borrow::Borrow<Option<<T as ArrowPrimitiveType>::Native>>,
- I: IntoIterator<Item = P>,
- {
- let iterator = iter.into_iter();
- let (_, upper) = iterator.size_hint();
- let len = upper.expect("trusted_len_unzip requires an upper limit");
-
- let (null, buffer) = trusted_len_unzip(iterator);
-
- let data =
- ArrayData::new(T::DATA_TYPE, len, None, Some(null), 0, vec![buffer], vec![]);
- PrimitiveArray::from(data)
- }
-}
-
-// TODO: the macro is needed here because we'd get "conflicting implementations" error
-// otherwise with both `From<Vec<T::Native>>` and `From<Vec<Option<T::Native>>>`.
-// We should revisit this in future.
-macro_rules! def_numeric_from_vec {
- ( $ty:ident ) => {
- impl From<Vec<<$ty as ArrowPrimitiveType>::Native>> for PrimitiveArray<$ty> {
- fn from(data: Vec<<$ty as ArrowPrimitiveType>::Native>) -> Self {
- let array_data = ArrayData::builder($ty::DATA_TYPE)
- .len(data.len())
- .add_buffer(Buffer::from_slice_ref(&data))
- .build();
- PrimitiveArray::from(array_data)
- }
- }
-
- // Constructs a primitive array from a vector. Should only be used for testing.
- impl From<Vec<Option<<$ty as ArrowPrimitiveType>::Native>>>
- for PrimitiveArray<$ty>
- {
- fn from(data: Vec<Option<<$ty as ArrowPrimitiveType>::Native>>) -> Self {
- PrimitiveArray::from_iter(data.iter())
- }
- }
- };
-}
-
-def_numeric_from_vec!(Int8Type);
-def_numeric_from_vec!(Int16Type);
-def_numeric_from_vec!(Int32Type);
-def_numeric_from_vec!(Int64Type);
-def_numeric_from_vec!(UInt8Type);
-def_numeric_from_vec!(UInt16Type);
-def_numeric_from_vec!(UInt32Type);
-def_numeric_from_vec!(UInt64Type);
-def_numeric_from_vec!(Float32Type);
-def_numeric_from_vec!(Float64Type);
-
-def_numeric_from_vec!(Date32Type);
-def_numeric_from_vec!(Date64Type);
-def_numeric_from_vec!(Time32SecondType);
-def_numeric_from_vec!(Time32MillisecondType);
-def_numeric_from_vec!(Time64MicrosecondType);
-def_numeric_from_vec!(Time64NanosecondType);
-def_numeric_from_vec!(IntervalYearMonthType);
-def_numeric_from_vec!(IntervalDayTimeType);
-def_numeric_from_vec!(DurationSecondType);
-def_numeric_from_vec!(DurationMillisecondType);
-def_numeric_from_vec!(DurationMicrosecondType);
-def_numeric_from_vec!(DurationNanosecondType);
-def_numeric_from_vec!(TimestampSecondType);
-def_numeric_from_vec!(TimestampMillisecondType);
-def_numeric_from_vec!(TimestampMicrosecondType);
-def_numeric_from_vec!(TimestampNanosecondType);
-
-impl<T: ArrowTimestampType> PrimitiveArray<T> {
- /// Construct a timestamp array from a vec of i64 values and an optional timezone
... 106505 lines suppressed ...
[arrow-datafusion] 03/03: Update DataFusion dependencies on arrow-rs
Posted by ag...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
commit bb110c3ff80f68838ec87ee7f7c1c6acd59dec04
Author: Andy Grove <an...@gmail.com>
AuthorDate: Sun Apr 18 10:21:34 2021 -0600
Update DataFusion dependencies on arrow-rs
---
rust/benchmarks/Cargo.toml | 4 ++--
rust/datafusion-examples/Cargo.toml | 4 ++--
rust/datafusion/Cargo.toml | 4 ++--
3 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/rust/benchmarks/Cargo.toml b/rust/benchmarks/Cargo.toml
index 5cdf0f9..7fd8444 100644
--- a/rust/benchmarks/Cargo.toml
+++ b/rust/benchmarks/Cargo.toml
@@ -31,8 +31,8 @@ simd = ["datafusion/simd"]
snmalloc = ["snmalloc-rs"]
[dependencies]
-arrow = { path = "../arrow" }
-parquet = { path = "../parquet" }
+arrow = { git = "https://github.com/apache/arrow-rs", rev = "e023b4c" }
+parquet = { git = "https://github.com/apache/arrow-rs", rev = "e023b4c" }
datafusion = { path = "../datafusion" }
structopt = { version = "0.3", default-features = false }
tokio = { version = "^1.0", features = ["macros", "rt", "rt-multi-thread"] }
diff --git a/rust/datafusion-examples/Cargo.toml b/rust/datafusion-examples/Cargo.toml
index c86e7cc..1a06050 100644
--- a/rust/datafusion-examples/Cargo.toml
+++ b/rust/datafusion-examples/Cargo.toml
@@ -29,10 +29,10 @@ publish = false
[dev-dependencies]
+arrow = { git = "https://github.com/apache/arrow-rs", rev = "e023b4c" }
+arrow-flight = { git = "https://github.com/apache/arrow-rs", rev = "e023b4c" }
datafusion = { path = "../datafusion" }
-arrow = { path = "../arrow" }
prost = "0.7"
-arrow-flight = { path = "../arrow-flight" }
tonic = "0.4"
tokio = { version = "1.0", features = ["macros", "rt", "rt-multi-thread", "sync"] }
futures = "0.3"
diff --git a/rust/datafusion/Cargo.toml b/rust/datafusion/Cargo.toml
index fd1c1b2..88a50dd 100644
--- a/rust/datafusion/Cargo.toml
+++ b/rust/datafusion/Cargo.toml
@@ -50,8 +50,8 @@ unicode_expressions = ["unicode-segmentation"]
[dependencies]
ahash = "0.7"
hashbrown = "0.11"
-arrow = { path = "../arrow", version = "4.0.0-SNAPSHOT", features = ["prettyprint"] }
-parquet = { path = "../parquet", version = "4.0.0-SNAPSHOT", features = ["arrow"] }
+arrow = { git = "https://github.com/apache/arrow-rs", rev = "e023b4c", features = ["prettyprint"] }
+parquet = { git = "https://github.com/apache/arrow-rs", rev = "e023b4c", features = ["arrow"] }
sqlparser = "0.9.0"
clap = "2.33"
rustyline = {version = "7.0", optional = true}
[arrow-datafusion] 01/03: Update Cargo.toml
Posted by ag...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
commit 479e1e2ce5deb9ba3653de5317e82bc29da0e13e
Author: Andy Grove <an...@gmail.com>
AuthorDate: Sun Apr 18 10:11:55 2021 -0600
Update Cargo.toml
---
rust/Cargo.toml | 8 +-------
1 file changed, 1 insertion(+), 7 deletions(-)
diff --git a/rust/Cargo.toml b/rust/Cargo.toml
index de26f87..0a4ef2a 100644
--- a/rust/Cargo.toml
+++ b/rust/Cargo.toml
@@ -17,18 +17,12 @@
[workspace]
members = [
- "arrow",
- "parquet",
- "parquet_derive",
- "parquet_derive_test",
"datafusion",
"datafusion-examples",
- "arrow-flight",
- "integration-testing",
"benchmarks",
]
# this package is excluded because it requires different compilation flags, thereby significantly changing
# how it is compiled within the workspace, causing the whole workspace to be compiled from scratch
# this way, this is a stand-alone package that compiles independently of the others.
-exclude = ["arrow-pyarrow-integration-testing", "ballista"]
+exclude = ["ballista"]