You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ze...@apache.org on 2025/06/25 20:33:19 UTC

(arrow-go) branch main updated: feat: expose Payload.WritePayload to allow serializing into IPC format (#421)

This is an automated email from the ASF dual-hosted git repository.

zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git


The following commit(s) were added to refs/heads/main by this push:
     new c542dd6  feat: expose Payload.WritePayload to allow serializing into IPC format (#421)
c542dd6 is described below

commit c542dd68e2757122ce8ffc15936f2df46664c30c
Author: Alvaro Viebrantz <al...@gmail.com>
AuthorDate: Wed Jun 25 16:33:14 2025 -0400

    feat: expose Payload.WritePayload to allow serializing into IPC format (#421)
    
    ### Rationale for this change
    The BigQuery Storage Write API now accepts Arrow data. But schema data
    and record batches needs to send separately. Right now the
    `ipc.NewWriter` writes the schema data to the output buffer, which makes
    it fail on the BigQuery side, as the first message is the schema. So we
    need a way to write just RecordBatches.
    
    See related discussion on
    https://github.com/googleapis/google-cloud-go/issues/12478
    
    ### What changes are included in this PR?
    Add `WritePayload ` method to `Payload`, which exposes the underlying
    `writeIPCPayload` method, allowing to write IPC messages more easily.
    
    ### Are these changes tested?
    Yes, tested with BigQuery Storage Write API. But also added local tests
    to it too.
    
    ### Are there any user-facing changes?
    Yes, `WritePayload` will be a public method available option to users.
---
 arrow/ipc/file_writer.go |  6 ++++++
 arrow/ipc/writer_test.go | 25 +++++++++++++++++++++++++
 2 files changed, 31 insertions(+)

diff --git a/arrow/ipc/file_writer.go b/arrow/ipc/file_writer.go
index b57cafb..2aa0c9b 100644
--- a/arrow/ipc/file_writer.go
+++ b/arrow/ipc/file_writer.go
@@ -203,6 +203,12 @@ func (p *Payload) SerializeBody(w io.Writer) error {
 	return nil
 }
 
+// WritePayload serializes the payload in IPC format
+// into the provided writer.
+func (p *Payload) WritePayload(w io.Writer) (int, error) {
+	return writeIPCPayload(w, *p)
+}
+
 func (p *Payload) Release() {
 	if p.meta != nil {
 		p.meta.Release()
diff --git a/arrow/ipc/writer_test.go b/arrow/ipc/writer_test.go
index dfc23a7..eea8f46 100644
--- a/arrow/ipc/writer_test.go
+++ b/arrow/ipc/writer_test.go
@@ -332,3 +332,28 @@ func TestGetPayloads(t *testing.T) {
 
 	assert.Truef(t, array.RecordEqual(rec, got), "expected: %s\ngot: %s", rec, got)
 }
+
+func TestWritePayload(t *testing.T) {
+	mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
+	defer mem.AssertSize(t, 0)
+
+	bldr := array.NewRecordBuilder(mem, arrow.NewSchema([]arrow.Field{{Name: "col", Type: arrow.PrimitiveTypes.Int8}}, nil))
+	bldr.Field(0).(*array.Int8Builder).AppendValues([]int8{1, 2, 3, 4, 5}, nil)
+	rec := bldr.NewRecord()
+	defer rec.Release()
+
+	var buf bytes.Buffer
+	p, err := GetRecordBatchPayload(rec, WithAllocator(mem))
+	defer p.Release()
+	require.NoError(t, err)
+
+	_, err = p.WritePayload(&buf)
+	require.NoError(t, err)
+
+	r := NewMessageReader(&buf, WithAllocator(mem))
+	defer r.Release()
+
+	msg, err := r.Message()
+	require.NoError(t, err)
+	require.True(t, msg.Type() == MessageRecordBatch)
+}