You are viewing a plain text version of this content. The canonical link for it is here.
Posted to github@arrow.apache.org by GitBox <gi...@apache.org> on 2022/10/26 02:04:21 UTC
[GitHub] [arrow-rs] tustvold opened a new pull request, #2929: Add lexsort benchmark (#2871)
tustvold opened a new pull request, #2929:
URL: https://github.com/apache/arrow-rs/pull/2929
# Which issue does this PR close?
<!--
We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes #123` indicates that this PR will close issue #123.
-->
Part of #2781
# Rationale for this change
<!--
Why are you proposing this change? If this is already explained clearly in the issue then this section is not needed.
Explaining clearly why changes are proposed helps reviewers understand your changes and offer better suggestions for fixes.
-->
Benchmarks good :smile:
# What changes are included in this PR?
<!--
There is no need to duplicate the description in the issue here but it is sometimes worth providing a summary of the individual changes in this PR.
-->
Adds some benchmarks of the row format, and adds a disclaimer to the lexsort kernels
```
lexsort_to_indices([i32, i32_opt]): 4096
time: [464.01 µs 464.15 µs 464.32 µs]
Found 3 outliers among 100 measurements (3.00%)
1 (1.00%) high mild
2 (2.00%) high severe
lexsort_rows([i32, i32_opt]): 4096
time: [429.55 µs 429.66 µs 429.78 µs]
Found 4 outliers among 100 measurements (4.00%)
2 (2.00%) high mild
2 (2.00%) high severe
lexsort_to_indices([i32, i32_opt]): 32768
time: [4.5412 ms 4.5443 ms 4.5486 ms]
Found 5 outliers among 100 measurements (5.00%)
2 (2.00%) high mild
3 (3.00%) high severe
lexsort_rows([i32, i32_opt]): 32768
time: [4.0447 ms 4.0460 ms 4.0474 ms]
Found 5 outliers among 100 measurements (5.00%)
3 (3.00%) high mild
2 (2.00%) high severe
lexsort_to_indices([i32, str_opt(16)]): 4096
time: [465.90 µs 466.07 µs 466.26 µs]
Found 6 outliers among 100 measurements (6.00%)
4 (4.00%) high mild
2 (2.00%) high severe
lexsort_rows([i32, str_opt(16)]): 4096
time: [500.10 µs 500.27 µs 500.49 µs]
Found 8 outliers among 100 measurements (8.00%)
2 (2.00%) high mild
6 (6.00%) high severe
lexsort_to_indices([i32, str_opt(16)]): 32768
time: [4.5679 ms 4.5693 ms 4.5707 ms]
Found 9 outliers among 100 measurements (9.00%)
8 (8.00%) high mild
1 (1.00%) high severe
lexsort_rows([i32, str_opt(16)]): 32768
time: [4.7611 ms 4.7641 ms 4.7671 ms]
Found 1 outliers among 100 measurements (1.00%)
1 (1.00%) high mild
lexsort_to_indices([i32, str(16)]): 4096
time: [466.06 µs 466.21 µs 466.36 µs]
Found 2 outliers among 100 measurements (2.00%)
2 (2.00%) high severe
lexsort_rows([i32, str(16)]): 4096
time: [391.45 µs 391.60 µs 391.76 µs]
Found 5 outliers among 100 measurements (5.00%)
5 (5.00%) high severe
lexsort_to_indices([i32, str(16)]): 32768
time: [4.5577 ms 4.5590 ms 4.5604 ms]
Found 6 outliers among 100 measurements (6.00%)
1 (1.00%) high mild
5 (5.00%) high severe
lexsort_rows([i32, str(16)]): 32768
time: [3.9101 ms 3.9132 ms 3.9162 ms]
lexsort_to_indices([str_opt(16), str(16)]): 4096
time: [878.19 µs 878.43 µs 878.72 µs]
Found 9 outliers among 100 measurements (9.00%)
4 (4.00%) high mild
5 (5.00%) high severe
lexsort_rows([str_opt(16), str(16)]): 4096
time: [461.13 µs 461.59 µs 462.23 µs]
Found 23 outliers among 100 measurements (23.00%)
23 (23.00%) high severe
lexsort_to_indices([str_opt(16), str(16)]): 32768
time: [9.0754 ms 9.0786 ms 9.0823 ms]
Found 8 outliers among 100 measurements (8.00%)
5 (5.00%) high mild
3 (3.00%) high severe
lexsort_rows([str_opt(16), str(16)]): 32768
time: [4.5031 ms 4.5072 ms 4.5113 ms]
lexsort_to_indices([str_opt(16), str_opt(50), str(16)]): 4096
time: [863.26 µs 863.49 µs 863.74 µs]
Found 6 outliers among 100 measurements (6.00%)
4 (4.00%) high mild
2 (2.00%) high severe
lexsort_rows([str_opt(16), str_opt(50), str(16)]): 4096
time: [537.53 µs 537.76 µs 537.99 µs]
Found 4 outliers among 100 measurements (4.00%)
4 (4.00%) high severe
lexsort_to_indices([str_opt(16), str_opt(50), str(16)]): 32768
time: [9.0009 ms 9.0051 ms 9.0098 ms]
Found 10 outliers among 100 measurements (10.00%)
6 (6.00%) high mild
4 (4.00%) high severe
lexsort_rows([str_opt(16), str_opt(50), str(16)]): 32768
time: [5.3922 ms 5.4006 ms 5.4092 ms]
lexsort_to_indices([str_opt(16), str(16), str_opt(16), str_opt(16), str_opt(16)]): 4096
time: [880.31 µs 880.52 µs 880.75 µs]
Found 4 outliers among 100 measurements (4.00%)
3 (3.00%) high mild
1 (1.00%) high severe
lexsort_rows([str_opt(16), str(16), str_opt(16), str_opt(16), str_opt(16)]): 4096
time: [686.41 µs 686.66 µs 686.94 µs]
Found 3 outliers among 100 measurements (3.00%)
1 (1.00%) high mild
2 (2.00%) high severe
lexsort_to_indices([str_opt(16), str(16), str_opt(16), str_opt(16), str_opt(16)]): 32768
time: [9.1124 ms 9.1163 ms 9.1207 ms]
Found 10 outliers among 100 measurements (10.00%)
4 (4.00%) high mild
6 (6.00%) high severe
lexsort_rows([str_opt(16), str(16), str_opt(16), str_opt(16), str_opt(16)]): 32768
time: [6.8218 ms 6.8290 ms 6.8362 ms]
lexsort_to_indices([i32_opt, dict(100,str_opt(50))]): 4096
time: [523.76 µs 523.95 µs 524.16 µs]
Found 8 outliers among 100 measurements (8.00%)
6 (6.00%) high mild
2 (2.00%) high severe
lexsort_rows([i32_opt, dict(100,str_opt(50))]): 4096
time: [430.36 µs 430.61 µs 430.90 µs]
Found 7 outliers among 100 measurements (7.00%)
4 (4.00%) high mild
3 (3.00%) high severe
lexsort_to_indices([i32_opt, dict(100,str_opt(50))]): 32768
time: [4.8896 ms 4.8922 ms 4.8950 ms]
Found 15 outliers among 100 measurements (15.00%)
13 (13.00%) high mild
2 (2.00%) high severe
lexsort_rows([i32_opt, dict(100,str_opt(50))]): 32768
time: [3.7030 ms 3.7046 ms 3.7063 ms]
Found 3 outliers among 100 measurements (3.00%)
3 (3.00%) high mild
lexsort_to_indices([dict(100,str_opt(50)), dict(100,str_opt(50))]): 4096
time: [153.02 µs 153.07 µs 153.11 µs]
Found 3 outliers among 100 measurements (3.00%)
1 (1.00%) high mild
2 (2.00%) high severe
lexsort_rows([dict(100,str_opt(50)), dict(100,str_opt(50))]): 4096
time: [200.52 µs 200.62 µs 200.73 µs]
Found 7 outliers among 100 measurements (7.00%)
3 (3.00%) high mild
4 (4.00%) high severe
Benchmarking lexsort_to_indices([dict(100,str_opt(50)), dict(100,str_opt(50))]): 32768: Warming up for 3.0000 s
Warning: Unable to complete 100 samples in 5.0s. You may wish to increase target time to 6.3s, enable flat sampling, or reduce sample count to 60.
lexsort_to_indices([dict(100,str_opt(50)), dict(100,str_opt(50))]): 32768
time: [1.2349 ms 1.2361 ms 1.2373 ms]
Found 3 outliers among 100 measurements (3.00%)
2 (2.00%) low mild
1 (1.00%) high severe
Benchmarking lexsort_rows([dict(100,str_opt(50)), dict(100,str_opt(50))]): 32768: Warming up for 3.0000 s
Warning: Unable to complete 100 samples in 5.0s. You may wish to increase target time to 7.4s, enable flat sampling, or reduce sample count to 50.
lexsort_rows([dict(100,str_opt(50)), dict(100,str_opt(50))]): 32768
time: [1.4587 ms 1.4594 ms 1.4601 ms]
Found 2 outliers among 100 measurements (2.00%)
1 (1.00%) high mild
1 (1.00%) high severe
Benchmarking lexsort_to_indices([dict(100,str_opt(50)), dict(100,str_opt(50)), dict(100,str_opt(50)), str(16)]): ...: Warming up for 3.0000 s
Warning: Unable to complete 100 samples in 5.0s. You may wish to increase target time to 7.3s, enable flat sampling, or reduce sample count to 50.
lexsort_to_indices([dict(100,str_opt(50)), dict(100,str_opt(50)), dict(100,str_opt(50)), str(16)]): ...
time: [1.4455 ms 1.4461 ms 1.4468 ms]
Found 11 outliers among 100 measurements (11.00%)
5 (5.00%) high mild
6 (6.00%) high severe
lexsort_rows([dict(100,str_opt(50)), dict(100,str_opt(50)), dict(100,str_opt(50)), str(16)]): 4096
time: [531.39 µs 531.58 µs 531.77 µs]
Found 6 outliers among 100 measurements (6.00%)
4 (4.00%) high mild
2 (2.00%) high severe
lexsort_to_indices([dict(100,str_opt(50)), dict(100,str_opt(50)), dict(100,str_opt(50)), str(16)]): ... #2
time: [15.592 ms 15.598 ms 15.604 ms]
Found 4 outliers among 100 measurements (4.00%)
3 (3.00%) high mild
1 (1.00%) high severe
lexsort_rows([dict(100,str_opt(50)), dict(100,str_opt(50)), dict(100,str_opt(50)), str(16)]): 32768
time: [4.7450 ms 4.7488 ms 4.7526 ms]
Found 1 outliers among 100 measurements (1.00%)
1 (1.00%) high mild
Benchmarking lexsort_to_indices([dict(100,str_opt(50)), dict(100,str_opt(50)), dict(100,str_opt(50)), str_opt(50)...: Warming up for 3.0000 s
Warning: Unable to complete 100 samples in 5.0s. You may wish to increase target time to 7.1s, enable flat sampling, or reduce sample count to 50.
lexsort_to_indices([dict(100,str_opt(50)), dict(100,str_opt(50)), dict(100,str_opt(50)), str_opt(50)...
time: [1.4102 ms 1.4107 ms 1.4113 ms]
Found 12 outliers among 100 measurements (12.00%)
5 (5.00%) high mild
7 (7.00%) high severe
lexsort_rows([dict(100,str_opt(50)), dict(100,str_opt(50)), dict(100,str_opt(50)), str_opt(50)]): 40...
time: [546.89 µs 547.06 µs 547.23 µs]
Found 7 outliers among 100 measurements (7.00%)
6 (6.00%) high mild
1 (1.00%) high severe
lexsort_to_indices([dict(100,str_opt(50)), dict(100,str_opt(50)), dict(100,str_opt(50)), str_opt(50)... #2
time: [15.753 ms 15.760 ms 15.768 ms]
Found 5 outliers among 100 measurements (5.00%)
5 (5.00%) high mild
lexsort_rows([dict(100,str_opt(50)), dict(100,str_opt(50)), dict(100,str_opt(50)), str_opt(50)]): 32...
time: [4.9877 ms 4.9912 ms 4.9947 ms]
Benchmarking lexsort_to_indices([dict(100,str_opt(50)), dict(100,str_opt(50)), dict(100,str_opt(50)), str_opt(50)... #3: Warming up for 3.0000 s
Warning: Unable to complete 100 samples in 5.0s. You may wish to increase target time to 7.1s, enable flat sampling, or reduce sample count to 50.
lexsort_to_indices([dict(100,str_opt(50)), dict(100,str_opt(50)), dict(100,str_opt(50)), str_opt(50)... #3
time: [1.4112 ms 1.4118 ms 1.4124 ms]
Found 8 outliers among 100 measurements (8.00%)
3 (3.00%) high mild
5 (5.00%) high severe
lexsort_rows([dict(100,str_opt(50)), dict(100,str_opt(50)), dict(100,str_opt(50)), str_opt(50)]): 40... #2
time: [547.35 µs 547.64 µs 547.99 µs]
Found 3 outliers among 100 measurements (3.00%)
3 (3.00%) high severe
lexsort_to_indices([dict(100,str_opt(50)), dict(100,str_opt(50)), dict(100,str_opt(50)), str_opt(50)... #4
time: [15.796 ms 15.804 ms 15.813 ms]
Found 5 outliers among 100 measurements (5.00%)
5 (5.00%) high mild
lexsort_rows([dict(100,str_opt(50)), dict(100,str_opt(50)), dict(100,str_opt(50)), str_opt(50)]): 32... #2
time: [5.0166 ms 5.0226 ms 5.0287 ms]
```
So sorting using the row format is in the same ballpark or significantly faster, with the performance benefit becoming more stark with more columns
# Are there any user-facing changes?
No
<!--
If there are user-facing changes then we may require documentation to be updated before approving the PR.
-->
<!---
If there are any breaking changes to public APIs, please add the `breaking change` label.
-->
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow-rs] tustvold commented on a diff in pull request #2929: Add lexsort benchmark (#2871)
Posted by GitBox <gi...@apache.org>.
tustvold commented on code in PR #2929:
URL: https://github.com/apache/arrow-rs/pull/2929#discussion_r1006197496
##########
arrow/benches/lexsort.rs:
##########
@@ -0,0 +1,171 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use arrow::compute::{lexsort_to_indices, SortColumn};
+use arrow::row::{RowConverter, SortField};
+use arrow::util::bench_util::{
+ create_dict_from_values, create_primitive_array, create_string_array_with_len,
+};
+use arrow_array::types::Int32Type;
+use arrow_array::{Array, ArrayRef, UInt32Array};
+use criterion::{criterion_group, criterion_main, Criterion};
+use std::sync::Arc;
+
+#[derive(Copy, Clone)]
+enum Column {
+ RequiredI32,
+ OptionalI32,
+ Required16CharString,
+ Optional16CharString,
+ Optional50CharString,
+ Optional100Value50CharStringDict,
+}
+
+impl std::fmt::Debug for Column {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ let s = match self {
+ Column::RequiredI32 => "i32",
+ Column::OptionalI32 => "i32_opt",
+ Column::Required16CharString => "str(16)",
+ Column::Optional16CharString => "str_opt(16)",
+ Column::Optional50CharString => "str_opt(50)",
+ Column::Optional100Value50CharStringDict => "dict(100,str_opt(50))",
+ };
+ f.write_str(s)
+ }
+}
+
+impl Column {
+ fn generate(self, size: usize) -> ArrayRef {
+ match self {
+ Column::RequiredI32 => {
+ Arc::new(create_primitive_array::<Int32Type>(size, 0.))
+ }
+ Column::OptionalI32 => {
+ Arc::new(create_primitive_array::<Int32Type>(size, 0.2))
+ }
+ Column::Required16CharString => {
+ Arc::new(create_string_array_with_len::<i32>(size, 0., 16))
+ }
+ Column::Optional16CharString => {
+ Arc::new(create_string_array_with_len::<i32>(size, 0.2, 16))
+ }
+ Column::Optional50CharString => {
+ Arc::new(create_string_array_with_len::<i32>(size, 0., 50))
+ }
+ Column::Optional100Value50CharStringDict => {
+ Arc::new(create_dict_from_values::<Int32Type>(
+ size,
+ 0.1,
+ &create_string_array_with_len::<i32>(100, 0., 50),
+ ))
+ }
+ }
+ }
+}
+
+fn do_bench(c: &mut Criterion, columns: &[Column], len: usize) {
+ let arrays: Vec<_> = columns.iter().map(|x| x.generate(len)).collect();
+ let sort_columns: Vec<_> = arrays
+ .iter()
+ .cloned()
+ .map(|values| SortColumn {
+ values,
+ options: None,
+ })
+ .collect();
+
+ c.bench_function(
+ &format!("lexsort_to_indices({:?}): {}", columns, len),
+ |b| {
+ b.iter(|| {
+ criterion::black_box(lexsort_to_indices(&sort_columns, None).unwrap())
+ })
+ },
+ );
+
+ c.bench_function(&format!("lexsort_rows({:?}): {}", columns, len), |b| {
Review Comment:
Yeah I wouldn't mind if there was some way to stop criterion truncating benchmark names, but I can't find such an option
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow-rs] tustvold commented on a diff in pull request #2929: Add lexsort benchmark (#2871)
Posted by GitBox <gi...@apache.org>.
tustvold commented on code in PR #2929:
URL: https://github.com/apache/arrow-rs/pull/2929#discussion_r1005139857
##########
arrow/benches/lexsort.rs:
##########
@@ -0,0 +1,166 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use arrow::compute::{lexsort_to_indices, SortColumn};
+use arrow::row::{RowConverter, SortField};
+use arrow::util::bench_util::{
+ create_dict_from_values, create_primitive_array, create_string_array_with_len,
+};
+use arrow_array::types::Int32Type;
+use arrow_array::{Array, ArrayRef, UInt32Array};
+use criterion::{criterion_group, criterion_main, Criterion};
+use std::sync::Arc;
+
+#[derive(Copy, Clone)]
+enum Column {
+ RequiredI32,
+ OptionalI32,
+ Required16CharString,
+ Optional16CharString,
+ Optional50CharString,
+ Optional100Value50CharStringDict,
+}
+
+impl std::fmt::Debug for Column {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ let s = match self {
+ Column::RequiredI32 => "i32",
+ Column::OptionalI32 => "i32_opt",
+ Column::Required16CharString => "str(16)",
+ Column::Optional16CharString => "str_opt(16)",
+ Column::Optional50CharString => "str_opt(50)",
+ Column::Optional100Value50CharStringDict => "dict(100,str_opt(50))",
+ };
+ f.write_str(s)
+ }
+}
+
+impl Column {
+ fn generate(self, size: usize) -> ArrayRef {
+ match self {
+ Column::RequiredI32 => {
+ Arc::new(create_primitive_array::<Int32Type>(size, 0.))
+ }
+ Column::OptionalI32 => {
+ Arc::new(create_primitive_array::<Int32Type>(size, 0.2))
+ }
+ Column::Required16CharString => {
+ Arc::new(create_string_array_with_len::<i32>(size, 0., 16))
+ }
+ Column::Optional16CharString => {
+ Arc::new(create_string_array_with_len::<i32>(size, 0.2, 16))
+ }
+ Column::Optional50CharString => {
+ Arc::new(create_string_array_with_len::<i32>(size, 0., 50))
+ }
+ Column::Optional100Value50CharStringDict => {
+ Arc::new(create_dict_from_values::<Int32Type>(
+ size,
+ 0.1,
+ &create_string_array_with_len::<i32>(100, 0., 50),
+ ))
+ }
+ }
+ }
+}
+
+fn do_bench(c: &mut Criterion, columns: &[Column], len: usize) {
+ let arrays: Vec<_> = columns.iter().map(|x| x.generate(len)).collect();
+ let sort_columns: Vec<_> = arrays
+ .iter()
+ .cloned()
+ .map(|values| SortColumn {
+ values,
+ options: None,
+ })
+ .collect();
+
+ c.bench_function(&format!("lexsort_to_indices({:?}): {}", columns, len), |b| {
+ b.iter(|| criterion::black_box(lexsort_to_indices(&sort_columns, None).unwrap()))
+ });
+
+ c.bench_function(&format!("lexsort_rows({:?}): {}", columns, len), |b| {
+ b.iter(|| {
+ criterion::black_box({
+ let fields = arrays
+ .iter()
+ .map(|a| SortField::new(a.data_type().clone()))
+ .collect();
+ let mut converter = RowConverter::new(fields);
+ let rows = converter.convert_columns(&arrays).unwrap();
+ let mut sort: Vec<_> = rows.iter().enumerate().collect();
+ sort.sort_unstable_by(|(_, a), (_, b)| a.cmp(b));
Review Comment:
I'm still a little bit confused as to why lexsort_to_indices can use sort_unstable whilst claiming to be a stable sort, but perhaps I've missed some subtlety. I just do the same thing here
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow-rs] tustvold commented on a diff in pull request #2929: Add lexsort benchmark (#2871)
Posted by GitBox <gi...@apache.org>.
tustvold commented on code in PR #2929:
URL: https://github.com/apache/arrow-rs/pull/2929#discussion_r1005139857
##########
arrow/benches/lexsort.rs:
##########
@@ -0,0 +1,166 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use arrow::compute::{lexsort_to_indices, SortColumn};
+use arrow::row::{RowConverter, SortField};
+use arrow::util::bench_util::{
+ create_dict_from_values, create_primitive_array, create_string_array_with_len,
+};
+use arrow_array::types::Int32Type;
+use arrow_array::{Array, ArrayRef, UInt32Array};
+use criterion::{criterion_group, criterion_main, Criterion};
+use std::sync::Arc;
+
+#[derive(Copy, Clone)]
+enum Column {
+ RequiredI32,
+ OptionalI32,
+ Required16CharString,
+ Optional16CharString,
+ Optional50CharString,
+ Optional100Value50CharStringDict,
+}
+
+impl std::fmt::Debug for Column {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ let s = match self {
+ Column::RequiredI32 => "i32",
+ Column::OptionalI32 => "i32_opt",
+ Column::Required16CharString => "str(16)",
+ Column::Optional16CharString => "str_opt(16)",
+ Column::Optional50CharString => "str_opt(50)",
+ Column::Optional100Value50CharStringDict => "dict(100,str_opt(50))",
+ };
+ f.write_str(s)
+ }
+}
+
+impl Column {
+ fn generate(self, size: usize) -> ArrayRef {
+ match self {
+ Column::RequiredI32 => {
+ Arc::new(create_primitive_array::<Int32Type>(size, 0.))
+ }
+ Column::OptionalI32 => {
+ Arc::new(create_primitive_array::<Int32Type>(size, 0.2))
+ }
+ Column::Required16CharString => {
+ Arc::new(create_string_array_with_len::<i32>(size, 0., 16))
+ }
+ Column::Optional16CharString => {
+ Arc::new(create_string_array_with_len::<i32>(size, 0.2, 16))
+ }
+ Column::Optional50CharString => {
+ Arc::new(create_string_array_with_len::<i32>(size, 0., 50))
+ }
+ Column::Optional100Value50CharStringDict => {
+ Arc::new(create_dict_from_values::<Int32Type>(
+ size,
+ 0.1,
+ &create_string_array_with_len::<i32>(100, 0., 50),
+ ))
+ }
+ }
+ }
+}
+
+fn do_bench(c: &mut Criterion, columns: &[Column], len: usize) {
+ let arrays: Vec<_> = columns.iter().map(|x| x.generate(len)).collect();
+ let sort_columns: Vec<_> = arrays
+ .iter()
+ .cloned()
+ .map(|values| SortColumn {
+ values,
+ options: None,
+ })
+ .collect();
+
+ c.bench_function(&format!("lexsort_to_indices({:?}): {}", columns, len), |b| {
+ b.iter(|| criterion::black_box(lexsort_to_indices(&sort_columns, None).unwrap()))
+ });
+
+ c.bench_function(&format!("lexsort_rows({:?}): {}", columns, len), |b| {
+ b.iter(|| {
+ criterion::black_box({
+ let fields = arrays
+ .iter()
+ .map(|a| SortField::new(a.data_type().clone()))
+ .collect();
+ let mut converter = RowConverter::new(fields);
+ let rows = converter.convert_columns(&arrays).unwrap();
+ let mut sort: Vec<_> = rows.iter().enumerate().collect();
+ sort.sort_unstable_by(|(_, a), (_, b)| a.cmp(b));
Review Comment:
I'm still a little bit confused as to why lexsort_to_indices can use sort_unstable whilst claiming to be a stable sort, but perhaps I've missed some subtlety
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow-rs] tustvold commented on a diff in pull request #2929: Add lexsort benchmark (#2871)
Posted by GitBox <gi...@apache.org>.
tustvold commented on code in PR #2929:
URL: https://github.com/apache/arrow-rs/pull/2929#discussion_r1005140128
##########
arrow/src/compute/kernels/sort.rs:
##########
@@ -942,11 +949,8 @@ pub fn lexsort_to_indices(
lexicographical_comparator.compare(a, b)
});
- Ok(UInt32Array::from(
- (&value_indices)[0..len]
- .iter()
- .map(|i| *i as u32)
- .collect::<Vec<u32>>(),
+ Ok(UInt32Array::from_iter_values(
Review Comment:
Drive by cleanup to avoid an intermediate array
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow-rs] Dandandan commented on a diff in pull request #2929: Add lexsort benchmark (#2871)
Posted by GitBox <gi...@apache.org>.
Dandandan commented on code in PR #2929:
URL: https://github.com/apache/arrow-rs/pull/2929#discussion_r1005558281
##########
arrow/benches/lexsort.rs:
##########
@@ -0,0 +1,166 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use arrow::compute::{lexsort_to_indices, SortColumn};
+use arrow::row::{RowConverter, SortField};
+use arrow::util::bench_util::{
+ create_dict_from_values, create_primitive_array, create_string_array_with_len,
+};
+use arrow_array::types::Int32Type;
+use arrow_array::{Array, ArrayRef, UInt32Array};
+use criterion::{criterion_group, criterion_main, Criterion};
+use std::sync::Arc;
+
+#[derive(Copy, Clone)]
+enum Column {
+ RequiredI32,
+ OptionalI32,
+ Required16CharString,
+ Optional16CharString,
+ Optional50CharString,
+ Optional100Value50CharStringDict,
+}
+
+impl std::fmt::Debug for Column {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ let s = match self {
+ Column::RequiredI32 => "i32",
+ Column::OptionalI32 => "i32_opt",
+ Column::Required16CharString => "str(16)",
+ Column::Optional16CharString => "str_opt(16)",
+ Column::Optional50CharString => "str_opt(50)",
+ Column::Optional100Value50CharStringDict => "dict(100,str_opt(50))",
+ };
+ f.write_str(s)
+ }
+}
+
+impl Column {
+ fn generate(self, size: usize) -> ArrayRef {
+ match self {
+ Column::RequiredI32 => {
+ Arc::new(create_primitive_array::<Int32Type>(size, 0.))
+ }
+ Column::OptionalI32 => {
+ Arc::new(create_primitive_array::<Int32Type>(size, 0.2))
+ }
+ Column::Required16CharString => {
+ Arc::new(create_string_array_with_len::<i32>(size, 0., 16))
+ }
+ Column::Optional16CharString => {
+ Arc::new(create_string_array_with_len::<i32>(size, 0.2, 16))
+ }
+ Column::Optional50CharString => {
+ Arc::new(create_string_array_with_len::<i32>(size, 0., 50))
+ }
+ Column::Optional100Value50CharStringDict => {
+ Arc::new(create_dict_from_values::<Int32Type>(
+ size,
+ 0.1,
+ &create_string_array_with_len::<i32>(100, 0., 50),
+ ))
+ }
+ }
+ }
+}
+
+fn do_bench(c: &mut Criterion, columns: &[Column], len: usize) {
+ let arrays: Vec<_> = columns.iter().map(|x| x.generate(len)).collect();
+ let sort_columns: Vec<_> = arrays
+ .iter()
+ .cloned()
+ .map(|values| SortColumn {
+ values,
+ options: None,
+ })
+ .collect();
+
+ c.bench_function(&format!("lexsort_to_indices({:?}): {}", columns, len), |b| {
+ b.iter(|| criterion::black_box(lexsort_to_indices(&sort_columns, None).unwrap()))
+ });
+
+ c.bench_function(&format!("lexsort_rows({:?}): {}", columns, len), |b| {
+ b.iter(|| {
+ criterion::black_box({
+ let fields = arrays
+ .iter()
+ .map(|a| SortField::new(a.data_type().clone()))
+ .collect();
+ let mut converter = RowConverter::new(fields);
+ let rows = converter.convert_columns(&arrays).unwrap();
+ let mut sort: Vec<_> = rows.iter().enumerate().collect();
+ sort.sort_unstable_by(|(_, a), (_, b)| a.cmp(b));
Review Comment:
I am not sure if the lexsort to indices is correctly doing stable sort. However it's possible to use unstable sort for stable sorting as long as you also sort the indexes:
https://rust-lang.github.io/rfcs/1884-unstable-sort.html
> Q: Can stable sort be performed using unstable sort?
A: Yes. If we transform [T] into [(T, usize)] by pairing every element with its index, then perform unstable sort, and finally remove indices, the result will be equivalent to stable sort.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow-rs] alamb commented on a diff in pull request #2929: Add lexsort benchmark (#2871)
Posted by GitBox <gi...@apache.org>.
alamb commented on code in PR #2929:
URL: https://github.com/apache/arrow-rs/pull/2929#discussion_r1005540154
##########
arrow/src/row/mod.rs:
##########
@@ -73,6 +73,24 @@
//! assert_eq!(&c2_values, &["a", "f", "c", "e"]);
//! ```
//!
+//! It can also be used to implement a fast lexicographic sort
Review Comment:
```suggestion
//! It can also be used to implement a fast multi-column / lexicographic sort
```
##########
arrow/benches/lexsort.rs:
##########
@@ -0,0 +1,171 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use arrow::compute::{lexsort_to_indices, SortColumn};
+use arrow::row::{RowConverter, SortField};
+use arrow::util::bench_util::{
+ create_dict_from_values, create_primitive_array, create_string_array_with_len,
+};
+use arrow_array::types::Int32Type;
+use arrow_array::{Array, ArrayRef, UInt32Array};
+use criterion::{criterion_group, criterion_main, Criterion};
+use std::sync::Arc;
+
+#[derive(Copy, Clone)]
+enum Column {
+ RequiredI32,
+ OptionalI32,
+ Required16CharString,
+ Optional16CharString,
+ Optional50CharString,
+ Optional100Value50CharStringDict,
+}
+
+impl std::fmt::Debug for Column {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ let s = match self {
+ Column::RequiredI32 => "i32",
+ Column::OptionalI32 => "i32_opt",
+ Column::Required16CharString => "str(16)",
+ Column::Optional16CharString => "str_opt(16)",
+ Column::Optional50CharString => "str_opt(50)",
+ Column::Optional100Value50CharStringDict => "dict(100,str_opt(50))",
+ };
+ f.write_str(s)
+ }
+}
+
+impl Column {
+ fn generate(self, size: usize) -> ArrayRef {
+ match self {
+ Column::RequiredI32 => {
+ Arc::new(create_primitive_array::<Int32Type>(size, 0.))
+ }
+ Column::OptionalI32 => {
+ Arc::new(create_primitive_array::<Int32Type>(size, 0.2))
+ }
+ Column::Required16CharString => {
+ Arc::new(create_string_array_with_len::<i32>(size, 0., 16))
+ }
+ Column::Optional16CharString => {
+ Arc::new(create_string_array_with_len::<i32>(size, 0.2, 16))
+ }
+ Column::Optional50CharString => {
+ Arc::new(create_string_array_with_len::<i32>(size, 0., 50))
+ }
+ Column::Optional100Value50CharStringDict => {
+ Arc::new(create_dict_from_values::<Int32Type>(
+ size,
+ 0.1,
+ &create_string_array_with_len::<i32>(100, 0., 50),
+ ))
+ }
+ }
+ }
+}
+
+fn do_bench(c: &mut Criterion, columns: &[Column], len: usize) {
+ let arrays: Vec<_> = columns.iter().map(|x| x.generate(len)).collect();
+ let sort_columns: Vec<_> = arrays
+ .iter()
+ .cloned()
+ .map(|values| SortColumn {
+ values,
+ options: None,
+ })
+ .collect();
+
+ c.bench_function(
+ &format!("lexsort_to_indices({:?}): {}", columns, len),
+ |b| {
+ b.iter(|| {
+ criterion::black_box(lexsort_to_indices(&sort_columns, None).unwrap())
+ })
+ },
+ );
+
+ c.bench_function(&format!("lexsort_rows({:?}): {}", columns, len), |b| {
Review Comment:
```suggestion
c.bench_function(&format!("RowFormat: lexsort_rows({:?}): {}", columns, len), |b| {
```
##########
arrow/benches/lexsort.rs:
##########
@@ -0,0 +1,166 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use arrow::compute::{lexsort_to_indices, SortColumn};
+use arrow::row::{RowConverter, SortField};
+use arrow::util::bench_util::{
+ create_dict_from_values, create_primitive_array, create_string_array_with_len,
+};
+use arrow_array::types::Int32Type;
+use arrow_array::{Array, ArrayRef, UInt32Array};
+use criterion::{criterion_group, criterion_main, Criterion};
+use std::sync::Arc;
+
+#[derive(Copy, Clone)]
+enum Column {
+ RequiredI32,
+ OptionalI32,
+ Required16CharString,
+ Optional16CharString,
+ Optional50CharString,
+ Optional100Value50CharStringDict,
+}
+
+impl std::fmt::Debug for Column {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ let s = match self {
+ Column::RequiredI32 => "i32",
+ Column::OptionalI32 => "i32_opt",
+ Column::Required16CharString => "str(16)",
+ Column::Optional16CharString => "str_opt(16)",
+ Column::Optional50CharString => "str_opt(50)",
+ Column::Optional100Value50CharStringDict => "dict(100,str_opt(50))",
+ };
+ f.write_str(s)
+ }
+}
+
+impl Column {
+ fn generate(self, size: usize) -> ArrayRef {
+ match self {
+ Column::RequiredI32 => {
+ Arc::new(create_primitive_array::<Int32Type>(size, 0.))
+ }
+ Column::OptionalI32 => {
+ Arc::new(create_primitive_array::<Int32Type>(size, 0.2))
+ }
+ Column::Required16CharString => {
+ Arc::new(create_string_array_with_len::<i32>(size, 0., 16))
+ }
+ Column::Optional16CharString => {
+ Arc::new(create_string_array_with_len::<i32>(size, 0.2, 16))
+ }
+ Column::Optional50CharString => {
+ Arc::new(create_string_array_with_len::<i32>(size, 0., 50))
+ }
+ Column::Optional100Value50CharStringDict => {
+ Arc::new(create_dict_from_values::<Int32Type>(
+ size,
+ 0.1,
+ &create_string_array_with_len::<i32>(100, 0., 50),
+ ))
+ }
+ }
+ }
+}
+
+fn do_bench(c: &mut Criterion, columns: &[Column], len: usize) {
+ let arrays: Vec<_> = columns.iter().map(|x| x.generate(len)).collect();
+ let sort_columns: Vec<_> = arrays
+ .iter()
+ .cloned()
+ .map(|values| SortColumn {
+ values,
+ options: None,
+ })
+ .collect();
+
+ c.bench_function(&format!("lexsort_to_indices({:?}): {}", columns, len), |b| {
+ b.iter(|| criterion::black_box(lexsort_to_indices(&sort_columns, None).unwrap()))
+ });
+
+ c.bench_function(&format!("lexsort_rows({:?}): {}", columns, len), |b| {
+ b.iter(|| {
+ criterion::black_box({
+ let fields = arrays
+ .iter()
+ .map(|a| SortField::new(a.data_type().clone()))
+ .collect();
+ let mut converter = RowConverter::new(fields);
+ let rows = converter.convert_columns(&arrays).unwrap();
+ let mut sort: Vec<_> = rows.iter().enumerate().collect();
+ sort.sort_unstable_by(|(_, a), (_, b)| a.cmp(b));
Review Comment:
I thought we changed lexsort_to_indices to be unstable in the name of performance. As in it shouldn't be claiming to be stable
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow-rs] tustvold merged pull request #2929: Add lexsort benchmark (#2871)
Posted by GitBox <gi...@apache.org>.
tustvold merged PR #2929:
URL: https://github.com/apache/arrow-rs/pull/2929
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow-rs] alamb commented on a diff in pull request #2929: Add lexsort benchmark (#2871)
Posted by GitBox <gi...@apache.org>.
alamb commented on code in PR #2929:
URL: https://github.com/apache/arrow-rs/pull/2929#discussion_r1006196885
##########
arrow/benches/lexsort.rs:
##########
@@ -0,0 +1,171 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use arrow::compute::{lexsort_to_indices, SortColumn};
+use arrow::row::{RowConverter, SortField};
+use arrow::util::bench_util::{
+ create_dict_from_values, create_primitive_array, create_string_array_with_len,
+};
+use arrow_array::types::Int32Type;
+use arrow_array::{Array, ArrayRef, UInt32Array};
+use criterion::{criterion_group, criterion_main, Criterion};
+use std::sync::Arc;
+
+#[derive(Copy, Clone)]
+enum Column {
+ RequiredI32,
+ OptionalI32,
+ Required16CharString,
+ Optional16CharString,
+ Optional50CharString,
+ Optional100Value50CharStringDict,
+}
+
+impl std::fmt::Debug for Column {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ let s = match self {
+ Column::RequiredI32 => "i32",
+ Column::OptionalI32 => "i32_opt",
+ Column::Required16CharString => "str(16)",
+ Column::Optional16CharString => "str_opt(16)",
+ Column::Optional50CharString => "str_opt(50)",
+ Column::Optional100Value50CharStringDict => "dict(100,str_opt(50))",
+ };
+ f.write_str(s)
+ }
+}
+
+impl Column {
+ fn generate(self, size: usize) -> ArrayRef {
+ match self {
+ Column::RequiredI32 => {
+ Arc::new(create_primitive_array::<Int32Type>(size, 0.))
+ }
+ Column::OptionalI32 => {
+ Arc::new(create_primitive_array::<Int32Type>(size, 0.2))
+ }
+ Column::Required16CharString => {
+ Arc::new(create_string_array_with_len::<i32>(size, 0., 16))
+ }
+ Column::Optional16CharString => {
+ Arc::new(create_string_array_with_len::<i32>(size, 0.2, 16))
+ }
+ Column::Optional50CharString => {
+ Arc::new(create_string_array_with_len::<i32>(size, 0., 50))
+ }
+ Column::Optional100Value50CharStringDict => {
+ Arc::new(create_dict_from_values::<Int32Type>(
+ size,
+ 0.1,
+ &create_string_array_with_len::<i32>(100, 0., 50),
+ ))
+ }
+ }
+ }
+}
+
+fn do_bench(c: &mut Criterion, columns: &[Column], len: usize) {
+ let arrays: Vec<_> = columns.iter().map(|x| x.generate(len)).collect();
+ let sort_columns: Vec<_> = arrays
+ .iter()
+ .cloned()
+ .map(|values| SortColumn {
+ values,
+ options: None,
+ })
+ .collect();
+
+ c.bench_function(
+ &format!("lexsort_to_indices({:?}): {}", columns, len),
+ |b| {
+ b.iter(|| {
+ criterion::black_box(lexsort_to_indices(&sort_columns, None).unwrap())
+ })
+ },
+ );
+
+ c.bench_function(&format!("lexsort_rows({:?}): {}", columns, len), |b| {
Review Comment:
classic tradeoff between concision and verboseness 😆
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow-rs] ursabot commented on pull request #2929: Add lexsort benchmark (#2871)
Posted by GitBox <gi...@apache.org>.
ursabot commented on PR #2929:
URL: https://github.com/apache/arrow-rs/pull/2929#issuecomment-1292663385
Benchmark runs are scheduled for baseline = 51d35684507a5a1a818bfb69497011f4e4593b9d and contender = 1d36bdf82853b344bab45dfda84f47ee7e3cfb3f. 1d36bdf82853b344bab45dfda84f47ee7e3cfb3f is a master commit associated with this PR. Results will be available as each benchmark for each run completes.
Conbench compare runs links:
[Skipped :warning: Benchmarking of arrow-rs-commits is not supported on ec2-t3-xlarge-us-east-2] [ec2-t3-xlarge-us-east-2](https://conbench.ursa.dev/compare/runs/49950c889f964a1f803f6383abd11a2c...289ea197ca3c4fb7943b1eaaea6d2687/)
[Skipped :warning: Benchmarking of arrow-rs-commits is not supported on test-mac-arm] [test-mac-arm](https://conbench.ursa.dev/compare/runs/afa4117c81a04553ac7e0db7a00ff5d3...2ba575528cc1421b879243c055fbaf34/)
[Skipped :warning: Benchmarking of arrow-rs-commits is not supported on ursa-i9-9960x] [ursa-i9-9960x](https://conbench.ursa.dev/compare/runs/89b347414a874a4bbe15b52fddf314fd...bc51e2f3b85a4400bb6c1a5d0ad3a90a/)
[Skipped :warning: Benchmarking of arrow-rs-commits is not supported on ursa-thinkcentre-m75q] [ursa-thinkcentre-m75q](https://conbench.ursa.dev/compare/runs/6dffea29903a49b29cd91740fa36417f...aba258fe42274b7bba6afa134d2e2acf/)
Buildkite builds:
Supported benchmarks:
ec2-t3-xlarge-us-east-2: Supported benchmark langs: Python, R. Runs only benchmarks with cloud = True
test-mac-arm: Supported benchmark langs: C++, Python, R
ursa-i9-9960x: Supported benchmark langs: Python, R, JavaScript
ursa-thinkcentre-m75q: Supported benchmark langs: C++, Java
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow-rs] tustvold commented on a diff in pull request #2929: Add lexsort benchmark (#2871)
Posted by GitBox <gi...@apache.org>.
tustvold commented on code in PR #2929:
URL: https://github.com/apache/arrow-rs/pull/2929#discussion_r1006024864
##########
arrow/benches/lexsort.rs:
##########
@@ -0,0 +1,171 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use arrow::compute::{lexsort_to_indices, SortColumn};
+use arrow::row::{RowConverter, SortField};
+use arrow::util::bench_util::{
+ create_dict_from_values, create_primitive_array, create_string_array_with_len,
+};
+use arrow_array::types::Int32Type;
+use arrow_array::{Array, ArrayRef, UInt32Array};
+use criterion::{criterion_group, criterion_main, Criterion};
+use std::sync::Arc;
+
+#[derive(Copy, Clone)]
+enum Column {
+ RequiredI32,
+ OptionalI32,
+ Required16CharString,
+ Optional16CharString,
+ Optional50CharString,
+ Optional100Value50CharStringDict,
+}
+
+impl std::fmt::Debug for Column {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ let s = match self {
+ Column::RequiredI32 => "i32",
+ Column::OptionalI32 => "i32_opt",
+ Column::Required16CharString => "str(16)",
+ Column::Optional16CharString => "str_opt(16)",
+ Column::Optional50CharString => "str_opt(50)",
+ Column::Optional100Value50CharStringDict => "dict(100,str_opt(50))",
+ };
+ f.write_str(s)
+ }
+}
+
+impl Column {
+ fn generate(self, size: usize) -> ArrayRef {
+ match self {
+ Column::RequiredI32 => {
+ Arc::new(create_primitive_array::<Int32Type>(size, 0.))
+ }
+ Column::OptionalI32 => {
+ Arc::new(create_primitive_array::<Int32Type>(size, 0.2))
+ }
+ Column::Required16CharString => {
+ Arc::new(create_string_array_with_len::<i32>(size, 0., 16))
+ }
+ Column::Optional16CharString => {
+ Arc::new(create_string_array_with_len::<i32>(size, 0.2, 16))
+ }
+ Column::Optional50CharString => {
+ Arc::new(create_string_array_with_len::<i32>(size, 0., 50))
+ }
+ Column::Optional100Value50CharStringDict => {
+ Arc::new(create_dict_from_values::<Int32Type>(
+ size,
+ 0.1,
+ &create_string_array_with_len::<i32>(100, 0., 50),
+ ))
+ }
+ }
+ }
+}
+
+fn do_bench(c: &mut Criterion, columns: &[Column], len: usize) {
+ let arrays: Vec<_> = columns.iter().map(|x| x.generate(len)).collect();
+ let sort_columns: Vec<_> = arrays
+ .iter()
+ .cloned()
+ .map(|values| SortColumn {
+ values,
+ options: None,
+ })
+ .collect();
+
+ c.bench_function(
+ &format!("lexsort_to_indices({:?}): {}", columns, len),
+ |b| {
+ b.iter(|| {
+ criterion::black_box(lexsort_to_indices(&sort_columns, None).unwrap())
+ })
+ },
+ );
+
+ c.bench_function(&format!("lexsort_rows({:?}): {}", columns, len), |b| {
Review Comment:
There is already an issue with the benchmark names being too long, so going to skip this one
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org