You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ko...@apache.org on 2025/10/14 12:43:23 UTC
(arrow-swift) branch main updated: fix: Padding now added to Arrow file marker and RecordBatches are being written with correct alignment (#95)
This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-swift.git
The following commit(s) were added to refs/heads/main by this push:
new 9259237 fix: Padding now added to Arrow file marker and RecordBatches are being written with correct alignment (#95)
9259237 is described below
commit 9259237fb5d0dede60136011b4e67efd834c5885
Author: willtemperley <wi...@gmail.com>
AuthorDate: Tue Oct 14 20:43:19 2025 +0800
fix: Padding now added to Arrow file marker and RecordBatches are being written with correct alignment (#95)
Files were being written with a non-padded file marker and alignment was
not being written when serializing record batches. Also the metadata
length was being set to zero in the block.
## What's Changed
The padded version of the filemarker is being written.
Padding is written to record blocks and record block metadata, i.e.:
```
addPadForAlignment(&writer)
```
The metadata length was being written as zero. This was preventing
PyArrow from reading files written by `ArrowWriter`. This has now been
calculated and set in the Block.
Closes #91.
---
Arrow/Sources/Arrow/ArrowWriter.swift | 22 ++++++++++++++++++----
1 file changed, 18 insertions(+), 4 deletions(-)
diff --git a/Arrow/Sources/Arrow/ArrowWriter.swift b/Arrow/Sources/Arrow/ArrowWriter.swift
index 3aa25b6..24176d3 100644
--- a/Arrow/Sources/Arrow/ArrowWriter.swift
+++ b/Arrow/Sources/Arrow/ArrowWriter.swift
@@ -144,12 +144,25 @@ public class ArrowWriter { // swiftlint:disable:this type_body_length
withUnsafeBytes(of: CONTINUATIONMARKER.littleEndian) {writer.append(Data($0))}
withUnsafeBytes(of: rbResult.1.o.littleEndian) {writer.append(Data($0))}
writer.append(rbResult.0)
+ addPadForAlignment(&writer)
+ let metadataLength = writer.count - startIndex
+ let bodyStart = writer.count
switch writeRecordBatchData(&writer, fields: batch.schema.fields, columns: batch.columns) {
case .success:
+ let bodyLength = writer.count - bodyStart
+ let expectedSize = startIndex + metadataLength + bodyLength
+ guard expectedSize == writer.count else {
+ return .failure(.invalid(
+ "Invalid Block. Expected \(expectedSize), got \(writer.count)"
+ ))
+ }
rbBlocks.append(
- org_apache_arrow_flatbuf_Block(offset: Int64(startIndex),
- metaDataLength: Int32(0),
- bodyLength: Int64(rbResult.1.o)))
+ org_apache_arrow_flatbuf_Block(
+ offset: Int64(startIndex),
+ metaDataLength: Int32(metadataLength),
+ bodyLength: Int64(bodyLength)
+ )
+ )
case .failure(let error):
return .failure(error)
}
@@ -293,6 +306,7 @@ public class ArrowWriter { // swiftlint:disable:this type_body_length
case .success(let schemaOffset):
fbb.finish(offset: schemaOffset)
writer.append(fbb.data)
+ addPadForAlignment(&writer)
case .failure(let error):
return .failure(error)
}
@@ -379,7 +393,7 @@ public class ArrowWriter { // swiftlint:disable:this type_body_length
addPadForAlignment(&markerData)
var writer: any DataWriter = FileDataWriter(fileHandle)
- writer.append(FILEMARKER.data(using: .utf8)!)
+ writer.append(markerData)
switch writeFile(&writer, info: info) {
case .success:
writer.append(FILEMARKER.data(using: .utf8)!)