You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by pa...@apache.org on 2022/08/05 18:22:48 UTC
[arrow-nanoarrow] branch main updated: Implement bitmap setters, getters, and element-wise builder (#10)
This is an automated email from the ASF dual-hosted git repository.
paleolimbot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-nanoarrow.git
The following commit(s) were added to refs/heads/main by this push:
new 51e5052 Implement bitmap setters, getters, and element-wise builder (#10)
51e5052 is described below
commit 51e5052ddd08fb424d8c20c86f9d5ea7d7b4ff51
Author: Dewey Dunnington <de...@fishandwhistle.net>
AuthorDate: Fri Aug 5 15:22:43 2022 -0300
Implement bitmap setters, getters, and element-wise builder (#10)
* initial sketch of bitmaps
* add tests
* test the builder
* move some typedefs to a separate header
* inline buffer functions
* inline bitmap functions
* add header guard
* header details
* some names to match arrow
* use arrow or arrow-inspired implementations
* remove pending values strategy from bitmap builder
* separate append and append unsafe tests
* rewrite countset using arrowish logic
* use a first byte-middle-bytes-last-bytes approach for int8 packer
* simplify unsafe int8 appender
* better int8 test
* copy the int8 packing for int32
* BitmapBuilder -> Bitmap
* Previous ArrowBitmap funcs -> ArrowBit funcs
* clarify bitmap doc
* add AppendUnsafe to match buffer
* fix the return value
* resize
---
CMakeLists.txt | 4 +-
src/nanoarrow/bitmap_inline.h | 323 ++++++++++++++++++++++++++++
src/nanoarrow/bitmap_test.cc | 276 ++++++++++++++++++++++++
src/nanoarrow/{buffer.c => buffer_inline.h} | 50 +++--
src/nanoarrow/nanoarrow.c | 1 -
src/nanoarrow/nanoarrow.h | 236 ++++++++------------
src/nanoarrow/typedefs_inline.h | 174 +++++++++++++++
7 files changed, 900 insertions(+), 164 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index e4a123a..b7f7663 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -29,7 +29,6 @@ include_directories(src)
add_library(
nanoarrow
src/nanoarrow/allocator.c
- src/nanoarrow/buffer.c
src/nanoarrow/error.c
src/nanoarrow/metadata.c
src/nanoarrow/schema.c
@@ -59,6 +58,7 @@ if (NANOARROW_BUILD_TESTS)
add_executable(allocator_test src/nanoarrow/allocator_test.cc)
add_executable(buffer_test src/nanoarrow/buffer_test.cc)
+ add_executable(bitmap_test src/nanoarrow/bitmap_test.cc)
add_executable(error_test src/nanoarrow/error_test.cc)
add_executable(metadata_test src/nanoarrow/metadata_test.cc)
add_executable(schema_test src/nanoarrow/schema_test.cc)
@@ -72,6 +72,7 @@ if (NANOARROW_BUILD_TESTS)
target_link_libraries(allocator_test nanoarrow GTest::gtest_main arrow_shared arrow_testing_shared)
target_link_libraries(buffer_test nanoarrow GTest::gtest_main)
+ target_link_libraries(bitmap_test nanoarrow GTest::gtest_main)
target_link_libraries(error_test nanoarrow GTest::gtest_main)
target_link_libraries(metadata_test nanoarrow GTest::gtest_main arrow_shared arrow_testing_shared)
target_link_libraries(schema_test nanoarrow GTest::gtest_main arrow_shared arrow_testing_shared)
@@ -80,6 +81,7 @@ if (NANOARROW_BUILD_TESTS)
include(GoogleTest)
gtest_discover_tests(allocator_test)
gtest_discover_tests(buffer_test)
+ gtest_discover_tests(bitmap_test)
gtest_discover_tests(error_test)
gtest_discover_tests(metadata_test)
gtest_discover_tests(schema_test)
diff --git a/src/nanoarrow/bitmap_inline.h b/src/nanoarrow/bitmap_inline.h
new file mode 100644
index 0000000..763da2a
--- /dev/null
+++ b/src/nanoarrow/bitmap_inline.h
@@ -0,0 +1,323 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef NANOARROW_BITMAP_INLINE_H_INCLUDED
+#define NANOARROW_BITMAP_INLINE_H_INCLUDED
+
+#include <stdlib.h>
+#include <string.h>
+
+#include "buffer_inline.h"
+#include "typedefs_inline.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+static const uint8_t _ArrowkBitmask[] = {1, 2, 4, 8, 16, 32, 64, 128};
+static const uint8_t _ArrowkFlippedBitmask[] = {254, 253, 251, 247, 239, 223, 191, 127};
+static const uint8_t _ArrowkPrecedingBitmask[] = {0, 1, 3, 7, 15, 31, 63, 127};
+static const uint8_t _ArrowkTrailingBitmask[] = {255, 254, 252, 248, 240, 224, 192, 128};
+
+static const uint8_t _ArrowkBytePopcount[] = {
+ 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3,
+ 4, 4, 5, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4,
+ 4, 5, 4, 5, 5, 6, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4,
+ 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5,
+ 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2,
+ 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5,
+ 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4,
+ 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 3, 4, 4, 5, 4, 5, 5, 6,
+ 4, 5, 5, 6, 5, 6, 6, 7, 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8};
+
+static inline int64_t _ArrowRoundUpToMultipleOf8(int64_t value) {
+ return (value + 7) & ~((int64_t)7);
+}
+
+static inline int64_t _ArrowRoundDownToMultipleOf8(int64_t value) {
+ return (value / 8) * 8;
+}
+
+static inline int64_t _ArrowBytesForBits(int64_t bits) {
+ return (bits >> 3) + ((bits & 7) != 0);
+}
+
+static inline void _ArrowBitmapPackInt8(const int8_t* values, uint8_t* out) {
+ *out = (values[0] | values[1] << 1 | values[2] << 2 | values[3] << 3 | values[4] << 4 |
+ values[5] << 5 | values[6] << 6 | values[7] << 7);
+}
+
+static inline void _ArrowBitmapPackInt32(const int32_t* values, uint8_t* out) {
+ *out = (values[0] | values[1] << 1 | values[2] << 2 | values[3] << 3 | values[4] << 4 |
+ values[5] << 5 | values[6] << 6 | values[7] << 7);
+}
+
+static inline int8_t ArrowBitGet(const uint8_t* bits, int64_t i) {
+ return (bits[i >> 3] >> (i & 0x07)) & 1;
+}
+
+static inline void ArrowBitSet(uint8_t* bits, int64_t i) {
+ bits[i / 8] |= _ArrowkBitmask[i % 8];
+}
+
+static inline void ArrowBitClear(uint8_t* bits, int64_t i) {
+ bits[i / 8] &= _ArrowkFlippedBitmask[i % 8];
+}
+
+static inline void ArrowBitSetTo(uint8_t* bits, int64_t i, uint8_t bit_is_set) {
+ bits[i / 8] ^=
+ ((uint8_t)(-((uint8_t)(bit_is_set != 0)) ^ bits[i / 8])) & _ArrowkBitmask[i % 8];
+}
+
+static inline void ArrowBitsSetTo(uint8_t* bits, int64_t start_offset, int64_t length,
+ uint8_t bits_are_set) {
+ const int64_t i_begin = start_offset;
+ const int64_t i_end = start_offset + length;
+ const uint8_t fill_byte = (uint8_t)(-bits_are_set);
+
+ const int64_t bytes_begin = i_begin / 8;
+ const int64_t bytes_end = i_end / 8 + 1;
+
+ const uint8_t first_byte_mask = _ArrowkPrecedingBitmask[i_begin % 8];
+ const uint8_t last_byte_mask = _ArrowkTrailingBitmask[i_end % 8];
+
+ if (bytes_end == bytes_begin + 1) {
+ // set bits within a single byte
+ const uint8_t only_byte_mask =
+ i_end % 8 == 0 ? first_byte_mask : (uint8_t)(first_byte_mask | last_byte_mask);
+ bits[bytes_begin] &= only_byte_mask;
+ bits[bytes_begin] |= (uint8_t)(fill_byte & ~only_byte_mask);
+ return;
+ }
+
+ // set/clear trailing bits of first byte
+ bits[bytes_begin] &= first_byte_mask;
+ bits[bytes_begin] |= (uint8_t)(fill_byte & ~first_byte_mask);
+
+ if (bytes_end - bytes_begin > 2) {
+ // set/clear whole bytes
+ memset(bits + bytes_begin + 1, fill_byte, (size_t)(bytes_end - bytes_begin - 2));
+ }
+
+ if (i_end % 8 == 0) {
+ return;
+ }
+
+ // set/clear leading bits of last byte
+ bits[bytes_end - 1] &= last_byte_mask;
+ bits[bytes_end - 1] |= (uint8_t)(fill_byte & ~last_byte_mask);
+}
+
+static inline int64_t ArrowBitCountSet(const uint8_t* bits, int64_t start_offset,
+ int64_t length) {
+ if (length == 0) {
+ return 0;
+ }
+
+ const int64_t i_begin = start_offset;
+ const int64_t i_end = start_offset + length;
+
+ const int64_t bytes_begin = i_begin / 8;
+ const int64_t bytes_end = i_end / 8 + 1;
+
+ const uint8_t first_byte_mask = _ArrowkPrecedingBitmask[i_begin % 8];
+ const uint8_t last_byte_mask = _ArrowkTrailingBitmask[i_end % 8];
+
+ if (bytes_end == bytes_begin + 1) {
+ // count bits within a single byte
+ const uint8_t only_byte_mask =
+ i_end % 8 == 0 ? first_byte_mask : (uint8_t)(first_byte_mask | last_byte_mask);
+ const uint8_t byte_masked = bits[bytes_begin] & only_byte_mask;
+ return _ArrowkBytePopcount[byte_masked];
+ }
+
+ int64_t count = 0;
+
+ // first byte
+ count += _ArrowkBytePopcount[bits[bytes_begin] & ~first_byte_mask];
+
+ // middle bytes
+ for (int64_t i = bytes_begin + 1; i < (bytes_end - 1); i++) {
+ count += _ArrowkBytePopcount[bits[i]];
+ }
+
+ // last byte
+ count += _ArrowkBytePopcount[bits[bytes_end - 1] & ~last_byte_mask];
+
+ return count;
+}
+
+static inline void ArrowBitmapInit(struct ArrowBitmap* bitmap) {
+ ArrowBufferInit(&bitmap->buffer);
+ bitmap->size_bits = 0;
+}
+
+static inline ArrowErrorCode ArrowBitmapReserve(struct ArrowBitmap* bitmap,
+ int64_t additional_size_bits) {
+ int64_t min_capacity_bits = bitmap->size_bits + additional_size_bits;
+ if (min_capacity_bits <= (bitmap->buffer.capacity_bytes * 8)) {
+ return NANOARROW_OK;
+ }
+
+ int result =
+ ArrowBufferReserve(&bitmap->buffer, _ArrowBytesForBits(additional_size_bits));
+ if (result != NANOARROW_OK) {
+ return result;
+ }
+
+ bitmap->buffer.data[bitmap->buffer.capacity_bytes - 1] = 0;
+ return NANOARROW_OK;
+}
+
+static inline ArrowErrorCode ArrowBitmapResize(struct ArrowBitmap* bitmap,
+ int64_t new_capacity_bits,
+ char shrink_to_fit) {
+ if (new_capacity_bits < 0) {
+ return EINVAL;
+ }
+
+ int64_t new_capacity_bytes = _ArrowBytesForBits(new_capacity_bits);
+ int result = ArrowBufferResize(&bitmap->buffer, new_capacity_bytes, shrink_to_fit);
+ if (result != NANOARROW_OK) {
+ return result;
+ }
+
+ if (new_capacity_bits < bitmap->size_bits) {
+ bitmap->size_bits = new_capacity_bits;
+ }
+
+ return NANOARROW_OK;
+}
+
+static inline ArrowErrorCode ArrowBitmapAppend(struct ArrowBitmap* bitmap,
+ uint8_t bits_are_set, int64_t length) {
+ int result = ArrowBitmapReserve(bitmap, length);
+ if (result != NANOARROW_OK) {
+ return result;
+ }
+
+ ArrowBitmapAppendUnsafe(bitmap, bits_are_set, length);
+ return NANOARROW_OK;
+}
+
+static inline void ArrowBitmapAppendUnsafe(struct ArrowBitmap* bitmap,
+ uint8_t bits_are_set, int64_t length) {
+ ArrowBitsSetTo(bitmap->buffer.data, bitmap->size_bits, length, bits_are_set);
+ bitmap->size_bits += length;
+ bitmap->buffer.size_bytes = _ArrowBytesForBits(bitmap->size_bits);
+}
+
+static inline void ArrowBitmapAppendInt8Unsafe(struct ArrowBitmap* bitmap,
+ const int8_t* values, int64_t n_values) {
+ if (n_values == 0) {
+ return;
+ }
+
+ const int8_t* values_cursor = values;
+ int64_t n_remaining = n_values;
+ int64_t out_i_cursor = bitmap->size_bits;
+ uint8_t* out_cursor = bitmap->buffer.data + bitmap->size_bits / 8;
+
+ // First byte
+ if ((out_i_cursor % 8) != 0) {
+ int64_t n_partial_bits = _ArrowRoundUpToMultipleOf8(out_i_cursor) - out_i_cursor;
+ for (int i = 0; i < n_partial_bits; i++) {
+ ArrowBitSetTo(bitmap->buffer.data, out_i_cursor++, values[i]);
+ }
+
+ out_cursor++;
+ values_cursor += n_partial_bits;
+ n_remaining -= n_partial_bits;
+ }
+
+ // Middle bytes
+ int64_t n_full_bytes = n_remaining / 8;
+ for (int64_t i = 0; i < n_full_bytes; i++) {
+ _ArrowBitmapPackInt8(values_cursor, out_cursor);
+ values_cursor += 8;
+ out_cursor++;
+ }
+
+ // Last byte
+ out_i_cursor += n_full_bytes * 8;
+ n_remaining -= n_full_bytes * 8;
+ if (n_remaining > 0) {
+ for (int i = 0; i < n_remaining; i++) {
+ ArrowBitSetTo(bitmap->buffer.data, out_i_cursor++, values_cursor[i]);
+ }
+ out_cursor++;
+ }
+
+ bitmap->size_bits += n_values;
+ bitmap->buffer.size_bytes = out_cursor - bitmap->buffer.data;
+}
+
+static inline void ArrowBitmapAppendInt32Unsafe(struct ArrowBitmap* bitmap,
+ const int32_t* values, int64_t n_values) {
+ if (n_values == 0) {
+ return;
+ }
+
+ const int32_t* values_cursor = values;
+ int64_t n_remaining = n_values;
+ int64_t out_i_cursor = bitmap->size_bits;
+ uint8_t* out_cursor = bitmap->buffer.data + bitmap->size_bits / 8;
+
+ // First byte
+ if ((out_i_cursor % 8) != 0) {
+ int64_t n_partial_bits = _ArrowRoundUpToMultipleOf8(out_i_cursor) - out_i_cursor;
+ for (int i = 0; i < n_partial_bits; i++) {
+ ArrowBitSetTo(bitmap->buffer.data, out_i_cursor++, values[i]);
+ }
+
+ out_cursor++;
+ values_cursor += n_partial_bits;
+ n_remaining -= n_partial_bits;
+ }
+
+ // Middle bytes
+ int64_t n_full_bytes = n_remaining / 8;
+ for (int64_t i = 0; i < n_full_bytes; i++) {
+ _ArrowBitmapPackInt32(values_cursor, out_cursor);
+ values_cursor += 8;
+ out_cursor++;
+ }
+
+ // Last byte
+ out_i_cursor += n_full_bytes * 8;
+ n_remaining -= n_full_bytes * 8;
+ if (n_remaining > 0) {
+ for (int i = 0; i < n_remaining; i++) {
+ ArrowBitSetTo(bitmap->buffer.data, out_i_cursor++, values_cursor[i]);
+ }
+ out_cursor++;
+ }
+
+ bitmap->size_bits += n_values;
+ bitmap->buffer.size_bytes = out_cursor - bitmap->buffer.data;
+}
+
+static inline void ArrowBitmapReset(struct ArrowBitmap* bitmap) {
+ ArrowBufferReset(&bitmap->buffer);
+ bitmap->size_bits = 0;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/nanoarrow/bitmap_test.cc b/src/nanoarrow/bitmap_test.cc
new file mode 100644
index 0000000..124857c
--- /dev/null
+++ b/src/nanoarrow/bitmap_test.cc
@@ -0,0 +1,276 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cstring>
+#include <string>
+
+#include <gtest/gtest.h>
+
+#include "nanoarrow/nanoarrow.h"
+
+TEST(BitmapTest, BitmapTestElement) {
+ uint8_t bitmap[10];
+
+ memset(bitmap, 0xff, sizeof(bitmap));
+ for (int i = 0; i < sizeof(bitmap) * 8; i++) {
+ EXPECT_EQ(ArrowBitGet(bitmap, i), 1);
+ }
+
+ bitmap[2] = 0xfd;
+ EXPECT_EQ(ArrowBitGet(bitmap, 16 + 0), 1);
+ EXPECT_EQ(ArrowBitGet(bitmap, 16 + 1), 0);
+ EXPECT_EQ(ArrowBitGet(bitmap, 16 + 2), 1);
+ EXPECT_EQ(ArrowBitGet(bitmap, 16 + 3), 1);
+ EXPECT_EQ(ArrowBitGet(bitmap, 16 + 4), 1);
+ EXPECT_EQ(ArrowBitGet(bitmap, 16 + 5), 1);
+ EXPECT_EQ(ArrowBitGet(bitmap, 16 + 6), 1);
+ EXPECT_EQ(ArrowBitGet(bitmap, 16 + 7), 1);
+
+ memset(bitmap, 0x00, sizeof(bitmap));
+ for (int i = 0; i < sizeof(bitmap) * 8; i++) {
+ EXPECT_EQ(ArrowBitGet(bitmap, i), 0);
+ }
+
+ bitmap[2] = 0x02;
+ EXPECT_EQ(ArrowBitGet(bitmap, 16 + 0), 0);
+ EXPECT_EQ(ArrowBitGet(bitmap, 16 + 1), 1);
+ EXPECT_EQ(ArrowBitGet(bitmap, 16 + 2), 0);
+ EXPECT_EQ(ArrowBitGet(bitmap, 16 + 3), 0);
+ EXPECT_EQ(ArrowBitGet(bitmap, 16 + 4), 0);
+ EXPECT_EQ(ArrowBitGet(bitmap, 16 + 5), 0);
+ EXPECT_EQ(ArrowBitGet(bitmap, 16 + 6), 0);
+ EXPECT_EQ(ArrowBitGet(bitmap, 16 + 7), 0);
+}
+
+TEST(BitmapTest, BitmapTestSetTo) {
+ uint8_t bitmap[10];
+
+ memset(bitmap, 0xff, sizeof(bitmap));
+ ArrowBitSetTo(bitmap, 16 + 1, 0);
+ EXPECT_EQ(bitmap[2], 0xfd);
+ ArrowBitSetTo(bitmap, 16 + 1, 1);
+ EXPECT_EQ(bitmap[2], 0xff);
+
+ memset(bitmap, 0xff, sizeof(bitmap));
+ ArrowBitClear(bitmap, 16 + 1);
+ EXPECT_EQ(bitmap[2], 0xfd);
+ ArrowBitSet(bitmap, 16 + 1);
+ EXPECT_EQ(bitmap[2], 0xff);
+
+ memset(bitmap, 0x00, sizeof(bitmap));
+ ArrowBitSetTo(bitmap, 16 + 1, 1);
+ EXPECT_EQ(bitmap[2], 0x02);
+ ArrowBitSetTo(bitmap, 16 + 1, 0);
+ EXPECT_EQ(bitmap[2], 0x00);
+
+ memset(bitmap, 0x00, sizeof(bitmap));
+ ArrowBitSet(bitmap, 16 + 1);
+ EXPECT_EQ(bitmap[2], 0x02);
+ ArrowBitClear(bitmap, 16 + 1);
+ EXPECT_EQ(bitmap[2], 0x00);
+}
+
+TEST(BitmapTest, BitmapTestCountSet) {
+ uint8_t bitmap[10];
+ memset(bitmap, 0x00, sizeof(bitmap));
+ ArrowBitSet(bitmap, 18);
+ ArrowBitSet(bitmap, 23);
+ ArrowBitSet(bitmap, 74);
+
+ EXPECT_EQ(ArrowBitCountSet(bitmap, 0, 80), 3);
+ EXPECT_EQ(ArrowBitCountSet(bitmap, 18, 57), 3);
+
+ EXPECT_EQ(ArrowBitCountSet(bitmap, 18, 0), 0);
+ EXPECT_EQ(ArrowBitCountSet(bitmap, 18, 1), 1);
+ EXPECT_EQ(ArrowBitCountSet(bitmap, 18, 2), 1);
+ EXPECT_EQ(ArrowBitCountSet(bitmap, 18, 3), 1);
+ EXPECT_EQ(ArrowBitCountSet(bitmap, 18, 4), 1);
+ EXPECT_EQ(ArrowBitCountSet(bitmap, 18, 5), 1);
+ EXPECT_EQ(ArrowBitCountSet(bitmap, 18, 6), 2);
+
+ EXPECT_EQ(ArrowBitCountSet(bitmap, 23, 1), 1);
+}
+
+TEST(BitmapTest, BitmapTestAppend) {
+ int8_t test_values[65];
+ memset(test_values, 0, sizeof(test_values));
+ test_values[4] = 1;
+ test_values[63] = 1;
+ test_values[64] = 1;
+
+ struct ArrowBitmap bitmap;
+ ArrowBitmapInit(&bitmap);
+
+ for (int64_t i = 0; i < 65; i++) {
+ ASSERT_EQ(ArrowBitmapAppend(&bitmap, test_values[i], 1), NANOARROW_OK);
+ }
+
+ EXPECT_EQ(bitmap.size_bits, 65);
+ EXPECT_EQ(ArrowBitGet(bitmap.buffer.data, 4), test_values[4]);
+ for (int i = 0; i < 65; i++) {
+ EXPECT_EQ(ArrowBitGet(bitmap.buffer.data, i), test_values[i]);
+ }
+
+ ArrowBitmapReset(&bitmap);
+}
+
+TEST(BitmapTest, BitmapTestResize) {
+ struct ArrowBitmap bitmap;
+ ArrowBitmapInit(&bitmap);
+
+ // Check normal usage, which is resize to the final length
+ // after appending a bunch of values
+ ArrowBitmapResize(&bitmap, 200, false);
+ EXPECT_EQ(bitmap.buffer.size_bytes, 0);
+ EXPECT_EQ(bitmap.buffer.capacity_bytes, 200 / 8);
+ EXPECT_EQ(bitmap.size_bits, 0);
+
+ ArrowBitmapAppendUnsafe(&bitmap, true, 100);
+ EXPECT_EQ(bitmap.buffer.size_bytes, 100 / 8 + 1);
+ EXPECT_EQ(bitmap.buffer.capacity_bytes, 200 / 8);
+ EXPECT_EQ(bitmap.size_bits, 100);
+
+ // Resize without shrinking
+ EXPECT_EQ(ArrowBitmapResize(&bitmap, 100, false), NANOARROW_OK);
+ EXPECT_EQ(bitmap.buffer.size_bytes, 100 / 8 + 1);
+ EXPECT_EQ(bitmap.buffer.capacity_bytes, 200 / 8);
+ EXPECT_EQ(bitmap.size_bits, 100);
+
+ // Resize with shrinking
+ EXPECT_EQ(ArrowBitmapResize(&bitmap, 100, true), NANOARROW_OK);
+ EXPECT_EQ(bitmap.buffer.size_bytes, 100 / 8 + 1);
+ EXPECT_EQ(bitmap.buffer.capacity_bytes, bitmap.buffer.size_bytes);
+ EXPECT_EQ(bitmap.size_bits, 100);
+
+ // Resize with shrinking when a reallocation isn't needed to shrink
+ EXPECT_EQ(ArrowBitmapResize(&bitmap, 99, true), NANOARROW_OK);
+ EXPECT_EQ(bitmap.buffer.size_bytes, 100 / 8 + 1);
+ EXPECT_EQ(bitmap.buffer.capacity_bytes, bitmap.buffer.size_bytes);
+ EXPECT_EQ(bitmap.size_bits, 99);
+
+ ArrowBitmapReset(&bitmap);
+}
+
+TEST(BitmapTest, BitmapTestAppendInt8Unsafe) {
+ struct ArrowBitmap bitmap;
+ ArrowBitmapInit(&bitmap);
+
+ // 68 because this will end in the middle of a byte, and appending twice
+ // will end exactly on the end of a byte
+ int8_t test_values[68];
+ memset(test_values, 0, sizeof(test_values));
+ // Make it easy to check the answer without repeating sequential packed byte values
+ for (int i = 0; i < 68; i++) {
+ test_values[i] = (i % 5) == 0;
+ }
+
+ // Append starting at 0
+ ASSERT_EQ(ArrowBitmapReserve(&bitmap, 68), NANOARROW_OK);
+ ArrowBitmapAppendInt8Unsafe(&bitmap, test_values, 68);
+
+ EXPECT_EQ(bitmap.size_bits, 68);
+ EXPECT_EQ(bitmap.buffer.size_bytes, 9);
+ for (int i = 0; i < 68; i++) {
+ EXPECT_EQ(ArrowBitGet(bitmap.buffer.data, i), test_values[i]);
+ }
+
+ // Append starting at a non-byte aligned value
+ ASSERT_EQ(ArrowBitmapReserve(&bitmap, 68), NANOARROW_OK);
+ ArrowBitmapAppendInt8Unsafe(&bitmap, test_values, 68);
+
+ EXPECT_EQ(bitmap.size_bits, 68 * 2);
+ EXPECT_EQ(bitmap.buffer.size_bytes, 17);
+ for (int i = 0; i < 68; i++) {
+ EXPECT_EQ(ArrowBitGet(bitmap.buffer.data, i), test_values[i]);
+ }
+ for (int i = 69; i < (68 * 2); i++) {
+ EXPECT_EQ(ArrowBitGet(bitmap.buffer.data, i), test_values[i - 68]);
+ }
+
+ // Append starting at a byte aligned but non-zero value
+ ASSERT_EQ(ArrowBitmapReserve(&bitmap, 68), NANOARROW_OK);
+ ArrowBitmapAppendInt8Unsafe(&bitmap, test_values, 68);
+
+ EXPECT_EQ(bitmap.size_bits, 204);
+ EXPECT_EQ(bitmap.buffer.size_bytes, 26);
+ for (int i = 0; i < 68; i++) {
+ EXPECT_EQ(ArrowBitGet(bitmap.buffer.data, i), test_values[i]);
+ }
+ for (int i = 69; i < 136; i++) {
+ EXPECT_EQ(ArrowBitGet(bitmap.buffer.data, i), test_values[i - 68]);
+ }
+ for (int i = 136; i < 204; i++) {
+ EXPECT_EQ(ArrowBitGet(bitmap.buffer.data, i), test_values[i - 136]);
+ }
+
+ ArrowBitmapReset(&bitmap);
+}
+
+TEST(BitmapTest, BitmapTestAppendInt32Unsafe) {
+ struct ArrowBitmap bitmap;
+ ArrowBitmapInit(&bitmap);
+
+ // 68 because this will end in the middle of a byte, and appending twice
+ // will end exactly on the end of a byte
+ int32_t test_values[68];
+ memset(test_values, 0, sizeof(test_values));
+ // Make it easy to check the answer without repeating sequential packed byte values
+ for (int i = 0; i < 68; i++) {
+ test_values[i] = (i % 5) == 0;
+ }
+
+ // Append starting at 0
+ ASSERT_EQ(ArrowBitmapReserve(&bitmap, 68), NANOARROW_OK);
+ ArrowBitmapAppendInt32Unsafe(&bitmap, test_values, 68);
+
+ EXPECT_EQ(bitmap.size_bits, 68);
+ EXPECT_EQ(bitmap.buffer.size_bytes, 9);
+ for (int i = 0; i < 68; i++) {
+ EXPECT_EQ(ArrowBitGet(bitmap.buffer.data, i), test_values[i]);
+ }
+
+ // Append starting at a non-byte aligned value
+ ASSERT_EQ(ArrowBitmapReserve(&bitmap, 68), NANOARROW_OK);
+ ArrowBitmapAppendInt32Unsafe(&bitmap, test_values, 68);
+
+ EXPECT_EQ(bitmap.size_bits, 68 * 2);
+ EXPECT_EQ(bitmap.buffer.size_bytes, 17);
+ for (int i = 0; i < 68; i++) {
+ EXPECT_EQ(ArrowBitGet(bitmap.buffer.data, i), test_values[i]);
+ }
+ for (int i = 69; i < (68 * 2); i++) {
+ EXPECT_EQ(ArrowBitGet(bitmap.buffer.data, i), test_values[i - 68]);
+ }
+
+ // Append starting at a byte aligned but non-zero value
+ ASSERT_EQ(ArrowBitmapReserve(&bitmap, 68), NANOARROW_OK);
+ ArrowBitmapAppendInt32Unsafe(&bitmap, test_values, 68);
+
+ EXPECT_EQ(bitmap.size_bits, 204);
+ EXPECT_EQ(bitmap.buffer.size_bytes, 26);
+ for (int i = 0; i < 68; i++) {
+ EXPECT_EQ(ArrowBitGet(bitmap.buffer.data, i), test_values[i]);
+ }
+ for (int i = 69; i < 136; i++) {
+ EXPECT_EQ(ArrowBitGet(bitmap.buffer.data, i), test_values[i - 68]);
+ }
+ for (int i = 136; i < 204; i++) {
+ EXPECT_EQ(ArrowBitGet(bitmap.buffer.data, i), test_values[i - 136]);
+ }
+
+ ArrowBitmapReset(&bitmap);
+}
diff --git a/src/nanoarrow/buffer.c b/src/nanoarrow/buffer_inline.h
similarity index 65%
rename from src/nanoarrow/buffer.c
rename to src/nanoarrow/buffer_inline.h
index ad56a98..a560304 100644
--- a/src/nanoarrow/buffer.c
+++ b/src/nanoarrow/buffer_inline.h
@@ -15,14 +15,20 @@
// specific language governing permissions and limitations
// under the License.
+#ifndef NANOARROW_BUFFER_INLINE_H_INCLUDED
+#define NANOARROW_BUFFER_INLINE_H_INCLUDED
+
#include <errno.h>
-#include <stddef.h>
-#include <stdlib.h>
+#include <stdint.h>
#include <string.h>
-#include "nanoarrow.h"
+#include "typedefs_inline.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
-static int64_t ArrowGrowByFactor(int64_t current_capacity, int64_t new_capacity) {
+static inline int64_t _ArrowGrowByFactor(int64_t current_capacity, int64_t new_capacity) {
int64_t doubled_capacity = current_capacity * 2;
if (doubled_capacity > new_capacity) {
return doubled_capacity;
@@ -31,15 +37,15 @@ static int64_t ArrowGrowByFactor(int64_t current_capacity, int64_t new_capacity)
}
}
-void ArrowBufferInit(struct ArrowBuffer* buffer) {
+static inline void ArrowBufferInit(struct ArrowBuffer* buffer) {
buffer->data = NULL;
buffer->size_bytes = 0;
buffer->capacity_bytes = 0;
buffer->allocator = ArrowBufferAllocatorDefault();
}
-ArrowErrorCode ArrowBufferSetAllocator(struct ArrowBuffer* buffer,
- struct ArrowBufferAllocator* allocator) {
+static inline ArrowErrorCode ArrowBufferSetAllocator(
+ struct ArrowBuffer* buffer, struct ArrowBufferAllocator* allocator) {
if (buffer->data == NULL) {
buffer->allocator = allocator;
return NANOARROW_OK;
@@ -48,7 +54,7 @@ ArrowErrorCode ArrowBufferSetAllocator(struct ArrowBuffer* buffer,
}
}
-void ArrowBufferReset(struct ArrowBuffer* buffer) {
+static inline void ArrowBufferReset(struct ArrowBuffer* buffer) {
if (buffer->data != NULL) {
buffer->allocator->free(buffer->allocator, (uint8_t*)buffer->data,
buffer->capacity_bytes);
@@ -59,14 +65,16 @@ void ArrowBufferReset(struct ArrowBuffer* buffer) {
buffer->size_bytes = 0;
}
-void ArrowBufferMove(struct ArrowBuffer* buffer, struct ArrowBuffer* buffer_out) {
+static inline void ArrowBufferMove(struct ArrowBuffer* buffer,
+ struct ArrowBuffer* buffer_out) {
memcpy(buffer_out, buffer, sizeof(struct ArrowBuffer));
buffer->data = NULL;
ArrowBufferReset(buffer);
}
-ArrowErrorCode ArrowBufferResize(struct ArrowBuffer* buffer, int64_t new_capacity_bytes,
- char shrink_to_fit) {
+static inline ArrowErrorCode ArrowBufferResize(struct ArrowBuffer* buffer,
+ int64_t new_capacity_bytes,
+ char shrink_to_fit) {
if (new_capacity_bytes < 0) {
return EINVAL;
}
@@ -91,27 +99,27 @@ ArrowErrorCode ArrowBufferResize(struct ArrowBuffer* buffer, int64_t new_capacit
return NANOARROW_OK;
}
-ArrowErrorCode ArrowBufferReserve(struct ArrowBuffer* buffer,
- int64_t additional_size_bytes) {
+static inline ArrowErrorCode ArrowBufferReserve(struct ArrowBuffer* buffer,
+ int64_t additional_size_bytes) {
int64_t min_capacity_bytes = buffer->size_bytes + additional_size_bytes;
if (min_capacity_bytes <= buffer->capacity_bytes) {
return NANOARROW_OK;
}
return ArrowBufferResize(
- buffer, ArrowGrowByFactor(buffer->capacity_bytes, min_capacity_bytes), 0);
+ buffer, _ArrowGrowByFactor(buffer->capacity_bytes, min_capacity_bytes), 0);
}
-void ArrowBufferAppendUnsafe(struct ArrowBuffer* buffer, const void* data,
- int64_t size_bytes) {
+static inline void ArrowBufferAppendUnsafe(struct ArrowBuffer* buffer, const void* data,
+ int64_t size_bytes) {
if (size_bytes > 0) {
memcpy(buffer->data + buffer->size_bytes, data, size_bytes);
buffer->size_bytes += size_bytes;
}
}
-ArrowErrorCode ArrowBufferAppend(struct ArrowBuffer* buffer, const void* data,
- int64_t size_bytes) {
+static inline ArrowErrorCode ArrowBufferAppend(struct ArrowBuffer* buffer,
+ const void* data, int64_t size_bytes) {
int result = ArrowBufferReserve(buffer, size_bytes);
if (result != NANOARROW_OK) {
return result;
@@ -120,3 +128,9 @@ ArrowErrorCode ArrowBufferAppend(struct ArrowBuffer* buffer, const void* data,
ArrowBufferAppendUnsafe(buffer, data, size_bytes);
return NANOARROW_OK;
}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/nanoarrow/nanoarrow.c b/src/nanoarrow/nanoarrow.c
index 90823aa..55103de 100644
--- a/src/nanoarrow/nanoarrow.c
+++ b/src/nanoarrow/nanoarrow.c
@@ -16,7 +16,6 @@
// under the License.
#include "allocator.c"
-#include "buffer.c"
#include "error.c"
#include "metadata.c"
#include "schema.c"
diff --git a/src/nanoarrow/nanoarrow.h b/src/nanoarrow/nanoarrow.h
index e1b8bd2..e4da6c5 100644
--- a/src/nanoarrow/nanoarrow.h
+++ b/src/nanoarrow/nanoarrow.h
@@ -22,97 +22,12 @@
#include <stdint.h>
#include <stdlib.h>
+#include "typedefs_inline.h"
+
#ifdef __cplusplus
extern "C" {
#endif
-// Extra guard for versions of Arrow without the canonical guard
-#ifndef ARROW_FLAG_DICTIONARY_ORDERED
-
-#ifndef ARROW_C_DATA_INTERFACE
-#define ARROW_C_DATA_INTERFACE
-
-#define ARROW_FLAG_DICTIONARY_ORDERED 1
-#define ARROW_FLAG_NULLABLE 2
-#define ARROW_FLAG_MAP_KEYS_SORTED 4
-
-struct ArrowSchema {
- // Array type description
- const char* format;
- const char* name;
- const char* metadata;
- int64_t flags;
- int64_t n_children;
- struct ArrowSchema** children;
- struct ArrowSchema* dictionary;
-
- // Release callback
- void (*release)(struct ArrowSchema*);
- // Opaque producer-specific data
- void* private_data;
-};
-
-struct ArrowArray {
- // Array data description
- int64_t length;
- int64_t null_count;
- int64_t offset;
- int64_t n_buffers;
- int64_t n_children;
- const void** buffers;
- struct ArrowArray** children;
- struct ArrowArray* dictionary;
-
- // Release callback
- void (*release)(struct ArrowArray*);
- // Opaque producer-specific data
- void* private_data;
-};
-
-#endif // ARROW_C_DATA_INTERFACE
-
-#ifndef ARROW_C_STREAM_INTERFACE
-#define ARROW_C_STREAM_INTERFACE
-
-struct ArrowArrayStream {
- // Callback to get the stream type
- // (will be the same for all arrays in the stream).
- //
- // Return value: 0 if successful, an `errno`-compatible error code otherwise.
- //
- // If successful, the ArrowSchema must be released independently from the stream.
- int (*get_schema)(struct ArrowArrayStream*, struct ArrowSchema* out);
-
- // Callback to get the next array
- // (if no error and the array is released, the stream has ended)
- //
- // Return value: 0 if successful, an `errno`-compatible error code otherwise.
- //
- // If successful, the ArrowArray must be released independently from the stream.
- int (*get_next)(struct ArrowArrayStream*, struct ArrowArray* out);
-
- // Callback to get optional detailed error information.
- // This must only be called if the last stream operation failed
- // with a non-0 return code.
- //
- // Return value: pointer to a null-terminated character array describing
- // the last error, or NULL if no description is available.
- //
- // The returned pointer is only valid until the next operation on this stream
- // (including release).
- const char* (*get_last_error)(struct ArrowArrayStream*);
-
- // Release callback: release the stream's own resources.
- // Note that arrays returned by `get_next` must be individually released.
- void (*release)(struct ArrowArrayStream*);
-
- // Opaque producer-specific data
- void* private_data;
-};
-
-#endif // ARROW_C_STREAM_INTERFACE
-#endif // ARROW_FLAG_DICTIONARY_ORDERED
-
/// \file Arrow C Implementation
///
/// EXPERIMENTAL. Interface subject to change.
@@ -142,26 +57,6 @@ void* ArrowRealloc(void* ptr, int64_t size);
/// \brief Free a pointer allocated using ArrowMalloc() or ArrowRealloc().
void ArrowFree(void* ptr);
-/// \brief Array buffer allocation and deallocation
-///
-/// Container for allocate, reallocate, and free methods that can be used
-/// to customize allocation and deallocation of buffers when constructing
-/// an ArrowArray.
-struct ArrowBufferAllocator {
- /// \brief Allocate a buffer or return NULL if it cannot be allocated
- uint8_t* (*allocate)(struct ArrowBufferAllocator* allocator, int64_t size);
-
- /// \brief Reallocate a buffer or return NULL if it cannot be reallocated
- uint8_t* (*reallocate)(struct ArrowBufferAllocator* allocator, uint8_t* ptr,
- int64_t old_size, int64_t new_size);
-
- /// \brief Deallocate a buffer allocated by this allocator
- void (*free)(struct ArrowBufferAllocator* allocator, uint8_t* ptr, int64_t size);
-
- /// \brief Opaque data specific to the allocator
- void* private_data;
-};
-
/// \brief Return the default allocator
///
/// The default allocator uses ArrowMalloc(), ArrowRealloc(), and
@@ -182,12 +77,6 @@ struct ArrowError {
char message[1024];
};
-/// \brief Return code for success.
-#define NANOARROW_OK 0
-
-/// \brief Represents an errno-compatible error code
-typedef int ArrowErrorCode;
-
/// \brief Set the contents of an error using printf syntax
ArrowErrorCode ArrowErrorSet(struct ArrowError* error, const char* fmt, ...);
@@ -483,49 +372,33 @@ ArrowErrorCode ArrowSchemaViewInit(struct ArrowSchemaView* schema_view,
/// }@
-/// \defgroup nanoarrow-buffer-builder Growable buffer builders
-
-/// \brief An owning mutable view of a buffer
-struct ArrowBuffer {
- /// \brief A pointer to the start of the buffer
- ///
- /// If capacity_bytes is 0, this value may be NULL.
- uint8_t* data;
-
- /// \brief The size of the buffer in bytes
- int64_t size_bytes;
-
- /// \brief The capacity of the buffer in bytes
- int64_t capacity_bytes;
-
- /// \brief The allocator that will be used to reallocate and/or free the buffer
- struct ArrowBufferAllocator* allocator;
-};
+/// \defgroup nanoarrow-buffer Owning, growable buffers
/// \brief Initialize an ArrowBuffer
///
/// Initialize a buffer with a NULL, zero-size buffer using the default
/// buffer allocator.
-void ArrowBufferInit(struct ArrowBuffer* buffer);
+static inline void ArrowBufferInit(struct ArrowBuffer* buffer);
/// \brief Set a newly-initialized buffer's allocator
///
/// Returns EINVAL if the buffer has already been allocated.
-ArrowErrorCode ArrowBufferSetAllocator(struct ArrowBuffer* buffer,
- struct ArrowBufferAllocator* allocator);
+static inline ArrowErrorCode ArrowBufferSetAllocator(
+ struct ArrowBuffer* buffer, struct ArrowBufferAllocator* allocator);
/// \brief Reset an ArrowBuffer
///
/// Releases the buffer using the allocator's free method if
/// the buffer's data member is non-null, sets the data member
/// to NULL, and sets the buffer's size and capacity to 0.
-void ArrowBufferReset(struct ArrowBuffer* buffer);
+static inline void ArrowBufferReset(struct ArrowBuffer* buffer);
/// \brief Move an ArrowBuffer
///
/// Transfers the buffer data and lifecycle management to another
/// address and resets buffer.
-void ArrowBufferMove(struct ArrowBuffer* buffer, struct ArrowBuffer* buffer_out);
+static inline void ArrowBufferMove(struct ArrowBuffer* buffer,
+ struct ArrowBuffer* buffer_out);
/// \brief Grow or shrink a buffer to a given capacity
///
@@ -533,32 +406,107 @@ void ArrowBufferMove(struct ArrowBuffer* buffer, struct ArrowBuffer* buffer_out)
/// if shrink_to_fit is non-zero. Calling ArrowBufferResize() does not
/// adjust the buffer's size member except to ensure that the invariant
/// capacity >= size remains true.
-ArrowErrorCode ArrowBufferResize(struct ArrowBuffer* buffer, int64_t new_capacity_bytes,
- char shrink_to_fit);
+static inline ArrowErrorCode ArrowBufferResize(struct ArrowBuffer* buffer,
+ int64_t new_capacity_bytes,
+ char shrink_to_fit);
/// \brief Ensure a buffer has at least a given additional capacity
///
/// Ensures that the buffer has space to append at least
/// additional_size_bytes, overallocating when required.
-ArrowErrorCode ArrowBufferReserve(struct ArrowBuffer* buffer,
- int64_t additional_size_bytes);
+static inline ArrowErrorCode ArrowBufferReserve(struct ArrowBuffer* buffer,
+ int64_t additional_size_bytes);
/// \brief Write data to buffer and increment the buffer size
///
/// This function does not check that buffer has the required capacity
-void ArrowBufferAppendUnsafe(struct ArrowBuffer* buffer, const void* data,
- int64_t size_bytes);
+static inline void ArrowBufferAppendUnsafe(struct ArrowBuffer* buffer, const void* data,
+ int64_t size_bytes);
/// \brief Write data to buffer and increment the buffer size
///
/// This function writes and ensures that the buffer has the required capacity,
/// possibly by reallocating the buffer. Like ArrowBufferReserve, this will
/// overallocate when reallocation is required.
-ArrowErrorCode ArrowBufferAppend(struct ArrowBuffer* buffer, const void* data,
- int64_t size_bytes);
+static inline ArrowErrorCode ArrowBufferAppend(struct ArrowBuffer* buffer,
+ const void* data, int64_t size_bytes);
+
+/// }@
+
+/// \defgroup nanoarrow-bitmap Bitmap utilities
+
+/// \brief Extract a boolean value from a bitmap
+static inline int8_t ArrowBitGet(const uint8_t* bits, int64_t i);
+
+/// \brief Set a boolean value to a bitmap to true
+static inline void ArrowBitSet(uint8_t* bits, int64_t i);
+
+/// \brief Set a boolean value to a bitmap to false
+static inline void ArrowBitClear(uint8_t* bits, int64_t i);
+
+/// \brief Set a boolean value to a bitmap
+static inline void ArrowBitSetTo(uint8_t* bits, int64_t i, uint8_t value);
+
+/// \brief Set a boolean value to a range in a bitmap
+static inline void ArrowBitsSetTo(uint8_t* bits, int64_t start_offset, int64_t length,
+ uint8_t bits_are_set);
+
+/// \brief Count true values in a bitmap
+static inline int64_t ArrowBitCountSet(const uint8_t* bits, int64_t i_from, int64_t i_to);
+
+/// \brief Initialize an ArrowBitmap
+///
+/// Initialize the builder's buffer, empty its cache, and reset the size to zero
+static inline void ArrowBitmapInit(struct ArrowBitmap* bitmap);
+
+/// \brief Ensure a bitmap builder has at least a given additional capacity
+///
+/// Ensures that the buffer has space to append at least
+/// additional_size_bits, overallocating when required.
+static inline ArrowErrorCode ArrowBitmapReserve(struct ArrowBitmap* bitmap,
+ int64_t additional_size_bits);
+
+/// \brief Grow or shrink a bitmap to a given capacity
+///
+/// When shrinking the capacity of the bitmap, the bitmap is only reallocated
+/// if shrink_to_fit is non-zero. Calling ArrowBitmapResize() does not
+/// adjust the buffer's size member except when shrinking new_capacity_bits
+/// to a value less than the current number of bits in the bitmap.
+static inline ArrowErrorCode ArrowBitmapResize(struct ArrowBitmap* bitmap,
+ int64_t new_capacity_bits,
+ char shrink_to_fit);
+
+/// \brief Reserve space for and append zero or more of the same boolean value to a bitmap
+static inline ArrowErrorCode ArrowBitmapAppend(struct ArrowBitmap* bitmap,
+ uint8_t bits_are_set, int64_t length);
+
+/// \brief Append zero or more of the same boolean value to a bitmap
+static inline void ArrowBitmapAppendUnsafe(struct ArrowBitmap* bitmap,
+ uint8_t bits_are_set, int64_t length);
+
+/// \brief Append boolean values encoded as int8_t to a bitmap
+///
+/// The values must all be 0 or 1.
+static inline void ArrowBitmapAppendInt8Unsafe(struct ArrowBitmap* bitmap,
+ const int8_t* values, int64_t n_values);
+
+/// \brief Append boolean values encoded as int32_t to a bitmap
+///
+/// The values must all be 0 or 1.
+static inline void ArrowBitmapAppendInt32Unsafe(struct ArrowBitmap* bitmap,
+ const int32_t* values, int64_t n_values);
+
+/// \brief Reset a bitmap builder
+///
+/// Releases any memory held by buffer, empties the cache, and resets the size to zero
+static inline void ArrowBitmapReset(struct ArrowBitmap* bitmap);
/// }@
+// Inline function definitions
+#include "bitmap_inline.h"
+#include "buffer_inline.h"
+
#ifdef __cplusplus
}
#endif
diff --git a/src/nanoarrow/typedefs_inline.h b/src/nanoarrow/typedefs_inline.h
new file mode 100644
index 0000000..2a30ba1
--- /dev/null
+++ b/src/nanoarrow/typedefs_inline.h
@@ -0,0 +1,174 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef NANOARROW_TYPEDEFS_INLINE_H_INCLUDED
+#define NANOARROW_TYPEDEFS_INLINE_H_INCLUDED
+
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/// \defgroup nanoarrow-inline-typedef Type definitions used in inlined implementations
+
+// Extra guard for versions of Arrow without the canonical guard
+#ifndef ARROW_FLAG_DICTIONARY_ORDERED
+
+#ifndef ARROW_C_DATA_INTERFACE
+#define ARROW_C_DATA_INTERFACE
+
+#define ARROW_FLAG_DICTIONARY_ORDERED 1
+#define ARROW_FLAG_NULLABLE 2
+#define ARROW_FLAG_MAP_KEYS_SORTED 4
+
+struct ArrowSchema {
+ // Array type description
+ const char* format;
+ const char* name;
+ const char* metadata;
+ int64_t flags;
+ int64_t n_children;
+ struct ArrowSchema** children;
+ struct ArrowSchema* dictionary;
+
+ // Release callback
+ void (*release)(struct ArrowSchema*);
+ // Opaque producer-specific data
+ void* private_data;
+};
+
+struct ArrowArray {
+ // Array data description
+ int64_t length;
+ int64_t null_count;
+ int64_t offset;
+ int64_t n_buffers;
+ int64_t n_children;
+ const void** buffers;
+ struct ArrowArray** children;
+ struct ArrowArray* dictionary;
+
+ // Release callback
+ void (*release)(struct ArrowArray*);
+ // Opaque producer-specific data
+ void* private_data;
+};
+
+#endif // ARROW_C_DATA_INTERFACE
+
+#ifndef ARROW_C_STREAM_INTERFACE
+#define ARROW_C_STREAM_INTERFACE
+
+struct ArrowArrayStream {
+ // Callback to get the stream type
+ // (will be the same for all arrays in the stream).
+ //
+ // Return value: 0 if successful, an `errno`-compatible error code otherwise.
+ //
+ // If successful, the ArrowSchema must be released independently from the stream.
+ int (*get_schema)(struct ArrowArrayStream*, struct ArrowSchema* out);
+
+ // Callback to get the next array
+ // (if no error and the array is released, the stream has ended)
+ //
+ // Return value: 0 if successful, an `errno`-compatible error code otherwise.
+ //
+ // If successful, the ArrowArray must be released independently from the stream.
+ int (*get_next)(struct ArrowArrayStream*, struct ArrowArray* out);
+
+ // Callback to get optional detailed error information.
+ // This must only be called if the last stream operation failed
+ // with a non-0 return code.
+ //
+ // Return value: pointer to a null-terminated character array describing
+ // the last error, or NULL if no description is available.
+ //
+ // The returned pointer is only valid until the next operation on this stream
+ // (including release).
+ const char* (*get_last_error)(struct ArrowArrayStream*);
+
+ // Release callback: release the stream's own resources.
+ // Note that arrays returned by `get_next` must be individually released.
+ void (*release)(struct ArrowArrayStream*);
+
+ // Opaque producer-specific data
+ void* private_data;
+};
+
+#endif // ARROW_C_STREAM_INTERFACE
+#endif // ARROW_FLAG_DICTIONARY_ORDERED
+
+/// \brief Return code for success.
+#define NANOARROW_OK 0
+
+/// \brief Represents an errno-compatible error code
+typedef int ArrowErrorCode;
+
+/// \brief Array buffer allocation and deallocation
+///
+/// Container for allocate, reallocate, and free methods that can be used
+/// to customize allocation and deallocation of buffers when constructing
+/// an ArrowArray.
+struct ArrowBufferAllocator {
+ /// \brief Allocate a buffer or return NULL if it cannot be allocated
+ uint8_t* (*allocate)(struct ArrowBufferAllocator* allocator, int64_t size);
+
+ /// \brief Reallocate a buffer or return NULL if it cannot be reallocated
+ uint8_t* (*reallocate)(struct ArrowBufferAllocator* allocator, uint8_t* ptr,
+ int64_t old_size, int64_t new_size);
+
+ /// \brief Deallocate a buffer allocated by this allocator
+ void (*free)(struct ArrowBufferAllocator* allocator, uint8_t* ptr, int64_t size);
+
+ /// \brief Opaque data specific to the allocator
+ void* private_data;
+};
+
+/// \brief An owning mutable view of a buffer
+struct ArrowBuffer {
+ /// \brief A pointer to the start of the buffer
+ ///
+ /// If capacity_bytes is 0, this value may be NULL.
+ uint8_t* data;
+
+ /// \brief The size of the buffer in bytes
+ int64_t size_bytes;
+
+ /// \brief The capacity of the buffer in bytes
+ int64_t capacity_bytes;
+
+ /// \brief The allocator that will be used to reallocate and/or free the buffer
+ struct ArrowBufferAllocator* allocator;
+};
+
+/// \brief An owning mutable view of a bitmap
+struct ArrowBitmap {
+ /// \brief An ArrowBuffer to hold the allocated memory
+ struct ArrowBuffer buffer;
+
+ /// \brief The number of bits that have been appended to the bitmap
+ int64_t size_bits;
+};
+
+/// }@
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif