You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ze...@apache.org on 2025/06/25 20:33:19 UTC
(arrow-go) branch main updated: feat: expose Payload.WritePayload to allow serializing into IPC format (#421)
This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new c542dd6 feat: expose Payload.WritePayload to allow serializing into IPC format (#421)
c542dd6 is described below
commit c542dd68e2757122ce8ffc15936f2df46664c30c
Author: Alvaro Viebrantz <al...@gmail.com>
AuthorDate: Wed Jun 25 16:33:14 2025 -0400
feat: expose Payload.WritePayload to allow serializing into IPC format (#421)
### Rationale for this change
The BigQuery Storage Write API now accepts Arrow data. But schema data
and record batches needs to send separately. Right now the
`ipc.NewWriter` writes the schema data to the output buffer, which makes
it fail on the BigQuery side, as the first message is the schema. So we
need a way to write just RecordBatches.
See related discussion on
https://github.com/googleapis/google-cloud-go/issues/12478
### What changes are included in this PR?
Add `WritePayload ` method to `Payload`, which exposes the underlying
`writeIPCPayload` method, allowing to write IPC messages more easily.
### Are these changes tested?
Yes, tested with BigQuery Storage Write API. But also added local tests
to it too.
### Are there any user-facing changes?
Yes, `WritePayload` will be a public method available option to users.
---
arrow/ipc/file_writer.go | 6 ++++++
arrow/ipc/writer_test.go | 25 +++++++++++++++++++++++++
2 files changed, 31 insertions(+)
diff --git a/arrow/ipc/file_writer.go b/arrow/ipc/file_writer.go
index b57cafb..2aa0c9b 100644
--- a/arrow/ipc/file_writer.go
+++ b/arrow/ipc/file_writer.go
@@ -203,6 +203,12 @@ func (p *Payload) SerializeBody(w io.Writer) error {
return nil
}
+// WritePayload serializes the payload in IPC format
+// into the provided writer.
+func (p *Payload) WritePayload(w io.Writer) (int, error) {
+ return writeIPCPayload(w, *p)
+}
+
func (p *Payload) Release() {
if p.meta != nil {
p.meta.Release()
diff --git a/arrow/ipc/writer_test.go b/arrow/ipc/writer_test.go
index dfc23a7..eea8f46 100644
--- a/arrow/ipc/writer_test.go
+++ b/arrow/ipc/writer_test.go
@@ -332,3 +332,28 @@ func TestGetPayloads(t *testing.T) {
assert.Truef(t, array.RecordEqual(rec, got), "expected: %s\ngot: %s", rec, got)
}
+
+func TestWritePayload(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+ defer mem.AssertSize(t, 0)
+
+ bldr := array.NewRecordBuilder(mem, arrow.NewSchema([]arrow.Field{{Name: "col", Type: arrow.PrimitiveTypes.Int8}}, nil))
+ bldr.Field(0).(*array.Int8Builder).AppendValues([]int8{1, 2, 3, 4, 5}, nil)
+ rec := bldr.NewRecord()
+ defer rec.Release()
+
+ var buf bytes.Buffer
+ p, err := GetRecordBatchPayload(rec, WithAllocator(mem))
+ defer p.Release()
+ require.NoError(t, err)
+
+ _, err = p.WritePayload(&buf)
+ require.NoError(t, err)
+
+ r := NewMessageReader(&buf, WithAllocator(mem))
+ defer r.Release()
+
+ msg, err := r.Message()
+ require.NoError(t, err)
+ require.True(t, msg.Type() == MessageRecordBatch)
+}