(iceberg-go) branch main updated: feat: implement object store location provider with hash-based path generation (#550)

[ze...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git


The following commit(s) were added to refs/heads/main by this push:
     new 0a56519  feat: implement object store location provider with hash-based path generation (#550)
0a56519 is described below

commit 0a565193d51faaf4c0566331a4443bbfc11e647f
Author: Blue Li <10...@users.noreply.github.com>
AuthorDate: Thu Sep 4 03:23:02 2025 +0800

    feat: implement object store location provider with hash-based path generation (#550)
    
    Implements hash-based data file path generation for object stores,
    following
    [Java](https://github.com/apache/iceberg/blob/5d5e0a3559946a94be2979f3ff9a09f3b8e67f19/core/src/main/java/org/apache/iceberg/LocationProviders.java#L128)/[Python](https://github.com/apache/iceberg-python/blob/4f02298ee7506470e3485a47eec7ffdc72f71bfc/pyiceberg/table/locations.py#L116)
    implementations.
    
    Note: Partition keys are not considered as parameters since
    `PartitionSpec.PartitionToPath()` handles this, and partitioned table
    writes are not yet supported.
---
 table/locations.go      | 50 +++++++++++++++++++++++++++++++++++++++++++++++++
 table/locations_test.go | 26 +++++++++++++++++++++++++
 2 files changed, 76 insertions(+)

diff --git a/table/locations.go b/table/locations.go
index 46478cb..4e47fcb 100644
--- a/table/locations.go
+++ b/table/locations.go
@@ -20,9 +20,19 @@ package table
 import (
 	"fmt"
 	"net/url"
+	"path"
+	"strconv"
+	"strings"
 
 	"github.com/apache/iceberg-go"
 	"github.com/google/uuid"
+	"github.com/twmb/murmur3"
+)
+
+const (
+	hashBinaryStringBits = 20
+	entropyDirLength     = 4
+	entropyDirDepth      = 3
 )
 
 type LocationProvider interface {
@@ -96,6 +106,46 @@ type objectStoreLocationProvider struct {
 	includePartitionPaths bool
 }
 
+func computeHash(dataFileName string) string {
+	// Bitwise AND to combat sign-extension; bitwise OR to preserve leading zeroes
+	topMask := 1 << hashBinaryStringBits
+	hashCode := int(murmur3.Sum32([]byte(dataFileName)))&(topMask-1) | topMask
+
+	// Convert to binary string and take the last hashBinaryStringBits
+	binaryStr := strconv.FormatInt(int64(hashCode), 2)
+
+	return dirsFromHash(binaryStr[len(binaryStr)-hashBinaryStringBits:])
+}
+
+func dirsFromHash(fileHash string) string {
+	// Divides hash into directories for optimized orphan removal operation
+	totalEntropyLength := entropyDirDepth * entropyDirLength
+
+	hashWithDirs := make([]string, 0)
+	for i := 0; i < totalEntropyLength; i += entropyDirLength {
+		hashWithDirs = append(hashWithDirs, fileHash[i:i+entropyDirLength])
+	}
+
+	if len(fileHash) > totalEntropyLength {
+		hashWithDirs = append(hashWithDirs, fileHash[totalEntropyLength:])
+	}
+
+	return strings.Join(hashWithDirs, "/")
+}
+
+func (p *objectStoreLocationProvider) NewDataLocation(dataFileName string) string {
+	if path.Dir(dataFileName) != "." {
+		return p.simpleLocationProvider.NewDataLocation(dataFileName)
+	}
+
+	hashedPath := computeHash(dataFileName)
+	if p.includePartitionPaths {
+		return p.simpleLocationProvider.dataPath.JoinPath(hashedPath, dataFileName).String()
+	} else {
+		return p.simpleLocationProvider.dataPath.JoinPath(hashedPath + "-" + dataFileName).String()
+	}
+}
+
 func newObjectStoreLocationProvider(tableLoc *url.URL, tableProps iceberg.Properties) (*objectStoreLocationProvider, error) {
 	slp, err := newSimpleLocationProvider(tableLoc, tableProps)
 	if err != nil {
diff --git a/table/locations_test.go b/table/locations_test.go
index 2ef2e53..8fa08bc 100644
--- a/table/locations_test.go
+++ b/table/locations_test.go
@@ -74,3 +74,29 @@ func TestLocationProviderMetadataFileLocationCustomPath(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, "s3://table-location/custom/path/00001-30313233-3435-4637-b839-616263646566.metadata.json", loc)
 }
+
+func TestObjectStoreLocationProvider(t *testing.T) {
+	provider, err := table.LoadLocationProvider("table_location",
+		iceberg.Properties{table.ObjectStoreEnabledKey: "true"})
+	require.NoError(t, err)
+
+	assert.Equal(t, "table_location/data/0101/0110/1001/10110010/a", provider.NewDataLocation("a"))
+	assert.Equal(t, "table_location/data/1110/0111/1110/00000011/b", provider.NewDataLocation("b"))
+	assert.Equal(t, "table_location/data/0010/1101/0110/01011111/c", provider.NewDataLocation("c"))
+	assert.Equal(t, "table_location/data/1001/0001/0100/01110011/d", provider.NewDataLocation("d"))
+}
+
+// TestObjectStoreLocationProviderPartitionedPathsDisabled tests that when partitioned paths are disabled,
+// the final "/" is still replaced with "-" even for un-partitioned files. This matches the behavior of
+// the Java implementation.
+func TestObjectStoreLocationProviderPartitionedPathsDisabled(t *testing.T) {
+	provider, err := table.LoadLocationProvider("table_location",
+		iceberg.Properties{table.ObjectStoreEnabledKey: "true", table.WriteObjectStorePartitionedPathsKey: "false"})
+	require.NoError(t, err)
+
+	assert.Equal(t, "table_location/data/0101/0110/1001/10110010-a", provider.NewDataLocation("a"))
+	assert.Equal(t, "table_location/data/1110/0111/1110/00000011-b", provider.NewDataLocation("b"))
+	assert.Equal(t, "table_location/data/0010/1101/0110/01011111-c", provider.NewDataLocation("c"))
+	assert.Equal(t, "table_location/data/1001/0001/0100/01110011-d", provider.NewDataLocation("d"))
+	assert.Equal(t, "table_location/data/0110/1010/0011/11101000-test.parquet", provider.NewDataLocation("test.parquet"))
+}