You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by al...@apache.org on 2021/12/20 16:46:29 UTC
[arrow-rs] branch master updated: Box RleDecoder index buffer (#1061) (#1062)
This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/master by this push:
new 127fb29 Box RleDecoder index buffer (#1061) (#1062)
127fb29 is described below
commit 127fb298fc1a48016e1ca409ccf57d1abb2c9d9c
Author: Raphael Taylor-Davies <17...@users.noreply.github.com>
AuthorDate: Mon Dec 20 16:46:23 2021 +0000
Box RleDecoder index buffer (#1061) (#1062)
* Box RleDecoder index buffer (#1061)
* Format
---
parquet/src/encodings/rle.rs | 15 ++++++++-------
1 file changed, 8 insertions(+), 7 deletions(-)
diff --git a/parquet/src/encodings/rle.rs b/parquet/src/encodings/rle.rs
index b2a23da..02a6034 100644
--- a/parquet/src/encodings/rle.rs
+++ b/parquet/src/encodings/rle.rs
@@ -319,7 +319,7 @@ pub struct RleDecoder {
bit_reader: Option<BitReader>,
// Buffer used when `bit_reader` is not `None`, for batch reading.
- index_buf: [i32; 1024],
+ index_buf: Option<Box<[i32; 1024]>>,
// The remaining number of values in RLE for this run
rle_left: u32,
@@ -338,7 +338,7 @@ impl RleDecoder {
rle_left: 0,
bit_packed_left: 0,
bit_reader: None,
- index_buf: [0; 1024],
+ index_buf: None,
current_value: None,
}
}
@@ -440,6 +440,8 @@ impl RleDecoder {
let mut values_read = 0;
while values_read < max_values {
+ let index_buf = self.index_buf.get_or_insert_with(|| Box::new([0; 1024]));
+
if self.rle_left > 0 {
let num_values =
cmp::min(max_values - values_read, self.rle_left as usize);
@@ -456,19 +458,18 @@ impl RleDecoder {
let mut num_values =
cmp::min(max_values - values_read, self.bit_packed_left as usize);
- num_values = cmp::min(num_values, self.index_buf.len());
+ num_values = cmp::min(num_values, index_buf.len());
loop {
num_values = bit_reader.get_batch::<i32>(
- &mut self.index_buf[..num_values],
+ &mut index_buf[..num_values],
self.bit_width as usize,
);
for i in 0..num_values {
- buffer[values_read + i]
- .clone_from(&dict[self.index_buf[i] as usize])
+ buffer[values_read + i].clone_from(&dict[index_buf[i] as usize])
}
self.bit_packed_left -= num_values as u32;
values_read += num_values;
- if num_values < self.index_buf.len() {
+ if num_values < index_buf.len() {
break;
}
}