You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by yu...@apache.org on 2025/04/15 07:40:36 UTC
(opendal) branch binding-go-reader-at updated: feat(bindings/go): add support for offset based read
This is an automated email from the ASF dual-hosted git repository.
yuchanns pushed a commit to branch binding-go-reader-at
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/binding-go-reader-at by this push:
new 7d3086bed feat(bindings/go): add support for offset based read
7d3086bed is described below
commit 7d3086bed8c087bd347270d03d82f1a1435c8153
Author: Hanchin Hsieh <me...@yuchanns.xyz>
AuthorDate: Tue Apr 15 15:40:22 2025 +0800
feat(bindings/go): add support for offset based read
---
bindings/c/include/opendal.h | 14 ++---
bindings/c/src/operator.rs | 21 +++----
bindings/go/ffi.go | 1 +
bindings/go/range.go | 76 ++++++++++++++++++++++++
bindings/go/reader.go | 73 ++++++++++++++++++++++-
bindings/go/tests/behavior_tests/opendal_test.go | 9 +++
bindings/go/tests/behavior_tests/read_test.go | 13 ++++
7 files changed, 184 insertions(+), 23 deletions(-)
diff --git a/bindings/c/include/opendal.h b/bindings/c/include/opendal.h
index e5b7f02d6..9d02bac55 100644
--- a/bindings/c/include/opendal.h
+++ b/bindings/c/include/opendal.h
@@ -866,13 +866,13 @@ struct opendal_result_read opendal_operator_read(const struct opendal_operator *
/**
* \brief Blocking read a range of data from `path`.
*
- * Read a range of data from `path` blocking by operator. The range starts from `offset`
- * with size of `size`.
+ * Read a range of data from `path` blocking by operator. The range starts from `start`
+ * to `end`.
*
* @param op The opendal_operator created previously
* @param path The path you want to read the data out
- * @param offset The start position of the range to read
- * @param size The size of the range to read
+ * @param start The start position of the range to read. 0 means the start of the file.
+ * @param end The end position of the range to read. 0 means the end of the file.
* @see opendal_operator
* @see opendal_result_read
* @see opendal_error
@@ -889,7 +889,7 @@ struct opendal_result_read opendal_operator_read(const struct opendal_operator *
* ```C
* // ... you have write "Hello, World!" to path "/testpath"
*
- * // Read 5 bytes starting from offset 0
+ * // Read bytes starting from 0 to 5
* opendal_result_read r = opendal_operator_read_range(op, "/testpath", 0, 5);
* assert(r.error == NULL);
*
@@ -910,8 +910,8 @@ struct opendal_result_read opendal_operator_read(const struct opendal_operator *
*/
struct opendal_result_read opendal_operator_read_range(const struct opendal_operator *op,
const char *path,
- uint64_t offset,
- uint64_t size);
+ uint64_t start,
+ uint64_t end);
/**
* \brief Blocking read the data from `path`.
diff --git a/bindings/c/src/operator.rs b/bindings/c/src/operator.rs
index 64e86ced2..6d53e148a 100644
--- a/bindings/c/src/operator.rs
+++ b/bindings/c/src/operator.rs
@@ -297,13 +297,13 @@ pub unsafe extern "C" fn opendal_operator_read(
/// \brief Blocking read a range of data from `path`.
///
-/// Read a range of data from `path` blocking by operator. The range starts from `offset`
-/// with size of `size`.
+/// Read a range of data from `path` blocking by operator. The range starts from `start`
+/// to `end`.
///
/// @param op The opendal_operator created previously
/// @param path The path you want to read the data out
-/// @param offset The start position of the range to read
-/// @param size The size of the range to read
+/// @param start The start position of the range to read. 0 means the start of the file.
+/// @param end The end position of the range to read. 0 means the end of the file.
/// @see opendal_operator
/// @see opendal_result_read
/// @see opendal_error
@@ -320,7 +320,7 @@ pub unsafe extern "C" fn opendal_operator_read(
/// ```C
/// // ... you have write "Hello, World!" to path "/testpath"
///
-/// // Read 5 bytes starting from offset 0
+/// // Read bytes starting from 0 to 5
/// opendal_result_read r = opendal_operator_read_range(op, "/testpath", 0, 5);
/// assert(r.error == NULL);
///
@@ -342,19 +342,14 @@ pub unsafe extern "C" fn opendal_operator_read(
pub unsafe extern "C" fn opendal_operator_read_range(
op: &opendal_operator,
path: *const c_char,
- offset: u64,
- size: u64,
+ start: u64,
+ end: u64,
) -> opendal_result_read {
assert!(!path.is_null());
let path = std::ffi::CStr::from_ptr(path)
.to_str()
.expect("malformed path");
- match op
- .deref()
- .read_with(path)
- .range(offset..offset + size)
- .call()
- {
+ match op.deref().read_with(path).range(start..end).call() {
Ok(b) => opendal_result_read {
data: opendal_bytes::new(b),
error: std::ptr::null_mut(),
diff --git a/bindings/go/ffi.go b/bindings/go/ffi.go
index 4cca16170..05a6ce21b 100644
--- a/bindings/go/ffi.go
+++ b/bindings/go/ffi.go
@@ -116,6 +116,7 @@ var withFFIs = []contextWithFFI{
withOperatorCreateDir,
withOperatorRead,
+ withOperatorReadRange,
withOperatorWrite,
withOperatorDelete,
withOperatorStat,
diff --git a/bindings/go/range.go b/bindings/go/range.go
new file mode 100644
index 000000000..b13d5f515
--- /dev/null
+++ b/bindings/go/range.go
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package opendal
+
+type BoundType int
+
+const (
+ Included BoundType = iota
+ Excluded
+ Unbounded
+)
+
+type Bound struct {
+ Type BoundType
+ Value uint64
+}
+
+func (b Bound) GetValue() uint64 {
+ switch b.Type {
+ case Included, Excluded:
+ return b.Value
+ default:
+ return 0
+ }
+}
+
+type Range struct {
+ start Bound
+ end Bound
+}
+
+func (r Range) Start() uint64 {
+ return r.start.GetValue()
+}
+
+func (r Range) End() uint64 {
+ return r.end.GetValue()
+}
+
+func rangeFrom(value uint64) *Range {
+ return &Range{
+ start: Bound{Type: Included, Value: value},
+ end: Bound{Type: Unbounded},
+ }
+}
+
+func rangeUntil(value uint64) *Range {
+ return &Range{
+ start: Bound{Type: Unbounded},
+ end: Bound{Type: Excluded, Value: value},
+ }
+}
+
+func rangeBetween(start, end uint64) *Range {
+ return &Range{
+ start: Bound{Type: Included, Value: start},
+ end: Bound{Type: Excluded, Value: end},
+ }
+}
diff --git a/bindings/go/reader.go b/bindings/go/reader.go
index 52810ddeb..23620e66b 100644
--- a/bindings/go/reader.go
+++ b/bindings/go/reader.go
@@ -27,6 +27,34 @@ import (
"github.com/jupiterrider/ffi"
)
+type readOption struct {
+ Range *Range
+}
+
+// withReadOption is a functional option type for configuring read options.
+type withReadOption func(*readOption)
+
+// ReadFrom creates a WithReadOption that specifies the starting position for reading.
+func ReadFrom(value uint64) withReadOption {
+ return func(o *readOption) {
+ o.Range = rangeFrom(value)
+ }
+}
+
+// ReadUntil creates a WithReadOption that specifies the ending position for reading.
+func ReadUntil(value uint64) withReadOption {
+ return func(o *readOption) {
+ o.Range = rangeUntil(value)
+ }
+}
+
+// ReadBetween creates a WithReadOption that specifies a range for reading.
+func ReadBetween(start, end uint64) withReadOption {
+ return func(o *readOption) {
+ o.Range = rangeBetween(start, end)
+ }
+}
+
// Read reads the entire contents of the file at the specified path into a byte slice.
//
// This function is a wrapper around the C-binding function `opendal_operator_read`.
@@ -57,9 +85,22 @@ import (
// }
//
// Note: This example assumes proper error handling and import statements.
-func (op *Operator) Read(path string) ([]byte, error) {
- read := getFFI[operatorRead](op.ctx, symOperatorRead)
- bytes, err := read(op.inner, path)
+func (op *Operator) Read(path string, opts ...withReadOption) ([]byte, error) {
+ var (
+ bytes opendalBytes
+ err error
+ )
+ var opt readOption
+ for _, o := range opts {
+ o(&opt)
+ }
+ if opt.Range != nil {
+ read := getFFI[operatorReadRange](op.ctx, symOperatorReadRange)
+ bytes, err = read(op.inner, path, opt.Range.Start(), opt.Range.End())
+ } else {
+ read := getFFI[operatorRead](op.ctx, symOperatorRead)
+ bytes, err = read(op.inner, path)
+ }
if err != nil {
return nil, err
}
@@ -223,6 +264,32 @@ var withOperatorRead = withFFI(ffiOpts{
}
})
+const symOperatorReadRange = "opendal_operator_read_range"
+
+type operatorReadRange func(op *opendalOperator, path string, start uint64, end uint64) (opendalBytes, error)
+
+var withOperatorReadRange = withFFI(ffiOpts{
+ sym: symOperatorReadRange,
+ rType: &typeResultRead,
+ aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer, &ffi.TypeUint64, &ffi.TypeUint64},
+}, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) operatorReadRange {
+ return func(op *opendalOperator, path string, start uint64, end uint64) (opendalBytes, error) {
+ bytePath, err := BytePtrFromString(path)
+ if err != nil {
+ return opendalBytes{}, err
+ }
+ var result resultRead
+ ffiCall(
+ unsafe.Pointer(&result),
+ unsafe.Pointer(&op),
+ unsafe.Pointer(&bytePath),
+ unsafe.Pointer(&start),
+ unsafe.Pointer(&end),
+ )
+ return result.data, parseError(ctx, result.error)
+ }
+})
+
const symOperatorReader = "opendal_operator_reader"
type operatorReader func(op *opendalOperator, path string) (*opendalReader, error)
diff --git a/bindings/go/tests/behavior_tests/opendal_test.go b/bindings/go/tests/behavior_tests/opendal_test.go
index 385186612..9f403160d 100644
--- a/bindings/go/tests/behavior_tests/opendal_test.go
+++ b/bindings/go/tests/behavior_tests/opendal_test.go
@@ -163,6 +163,15 @@ func genFixedBytes(size uint) []byte {
return content
}
+func genOffsetLength(size uint) (uint64, uint64) {
+ n, _ := rand.Int(rand.Reader, big.NewInt(int64(size-1)))
+ offset := uint64(n.Int64())
+ n, _ = rand.Int(rand.Reader, big.NewInt(int64(uint64(size)-offset)))
+ length := uint64(n.Int64()) + 1
+
+ return offset, length
+}
+
type fixture struct {
op *opendal.Operator
lock *sync.Mutex
diff --git a/bindings/go/tests/behavior_tests/read_test.go b/bindings/go/tests/behavior_tests/read_test.go
index ff125cf1d..06aedbbe1 100644
--- a/bindings/go/tests/behavior_tests/read_test.go
+++ b/bindings/go/tests/behavior_tests/read_test.go
@@ -31,6 +31,7 @@ func testsRead(cap *opendal.Capability) []behaviorTest {
}
return []behaviorTest{
testReadFull,
+ testReadRange,
testReader,
testReadNotExist,
testReadWithDirPath,
@@ -49,6 +50,18 @@ func testReadFull(assert *require.Assertions, op *opendal.Operator, fixture *fix
assert.Equal(content, bs, "read content")
}
+func testReadRange(assert *require.Assertions, op *opendal.Operator, fixture *fixture) {
+ path, content, size := fixture.NewFile()
+
+ assert.Nil(op.Write(path, content), "write must succeed")
+ offset, length := genOffsetLength(size)
+
+ bs, err := op.Read(path, opendal.ReadBetween(offset, offset+length))
+ assert.Nil(err)
+ assert.Equal(length, uint64(len(bs)), "read size")
+ assert.Equal(content[offset:offset+length], bs, "read content")
+}
+
func testReader(assert *require.Assertions, op *opendal.Operator, fixture *fixture) {
path, content, size := fixture.NewFile()