You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by at...@apache.org on 2026/06/25 08:00:45 UTC
(iceberg-go) branch main updated: fix(table): harden GeoArrow CRS read parsing (#1239)
This is an automated email from the ASF dual-hosted git repository.
laskoviymishka pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new 447c3460 fix(table): harden GeoArrow CRS read parsing (#1239)
447c3460 is described below
commit 447c346020d2f010aa01af6bd711c67b495e9446
Author: Revanth <re...@gmail.com>
AuthorDate: Thu Jun 25 03:00:40 2026 -0500
fix(table): harden GeoArrow CRS read parsing (#1239)
closes #1231
## Summary
- Reject unsupported GeoArrow string CRS encodings for `projjson` and
WKT2:2019.
- Keep non-empty string CRS values readable to preserve existing data
and read/write round-tripping.
- Preserve canonical `OGC:CRS84` / `EPSG:4326` handling even when
`crs_type:"srid"` is present.
- Improve JSON-object CRS diagnostics for invalid `id`, `id.authority`,
and `id.code` members.
## Testing
- go test ./table -run
'Test(IcebergGeoTypesToArrowSchema|GeoArrowCRSToIcebergCRS|IcebergCRSToGeoArrowMetadata)'
- go test ./table
---
table/arrow_utils.go | 89 +++++++++++++++++++----
table/arrow_utils_internal_test.go | 140 +++++++++++++++++++++++++++++++++++++
table/arrow_utils_test.go | 24 ++++++-
3 files changed, 237 insertions(+), 16 deletions(-)
diff --git a/table/arrow_utils.go b/table/arrow_utils.go
index 0f64cd5f..fef85387 100644
--- a/table/arrow_utils.go
+++ b/table/arrow_utils.go
@@ -1909,6 +1909,49 @@ func checkCRSJSON(rawCrs json.RawMessage) bool {
return len(b) > 0 && b[0] == '{'
}
+// errWKT2CRSNotSupported is returned for WKT2:2019 CRS definitions, which this
+// package does not support yet.
+var errWKT2CRSNotSupported = errors.New("CRS type wkt2:2019 not supported")
+
+// errPROJJSONStringCRSNotSupported is returned for projjson-typed string CRS
+// values, which this package does not support yet.
+var errPROJJSONStringCRSNotSupported = errors.New("CRS type projjson not supported for string CRS")
+
+// isWKT2CRSString reports whether crs looks like a WKT2 (ISO 19162) CRS
+// definition. It is a best-effort heuristic used only to surface the clearer
+// errWKT2CRSNotSupported message. The list covers the WKT2:2019 CRS keywords
+// and their long forms.
+func isWKT2CRSString(crs string) bool {
+ upper := strings.ToUpper(strings.TrimSpace(crs))
+
+ for _, prefix := range []string{
+ "BOUNDCRS[",
+ "COMPOUNDCRS[",
+ "COORDINATEMETADATA[",
+ "COORDINATEOPERATION[",
+ "CRS[",
+ "DERIVEDPROJCRS[",
+ "ENGCRS[",
+ "ENGINEERINGCRS[",
+ "GEODCRS[",
+ "GEODETICCRS[",
+ "GEOGCRS[",
+ "GEOGRAPHICCRS[",
+ "PARAMETRICCRS[",
+ "PROJCRS[",
+ "PROJECTEDCRS[",
+ "TIMECRS[",
+ "VERTCRS[",
+ "VERTICALCRS[",
+ } {
+ if strings.HasPrefix(upper, prefix) {
+ return true
+ }
+ }
+
+ return false
+}
+
func geoArrowCRSToIcebergCRS(meta geoarrow.Metadata) (string, error) {
if len(meta.CRS) == 0 {
return "srid:0", nil
@@ -1921,23 +1964,34 @@ func geoArrowCRSToIcebergCRS(meta geoarrow.Metadata) (string, error) {
if err := json.Unmarshal(meta.CRS, &crs); err != nil {
return "", fmt.Errorf("invalid geoarrow CRS metadata: %w", err)
}
+ if crs == "" {
+ return "", errors.New("unsupported CRS: empty string CRS")
+ }
if strings.EqualFold(crs, "OGC:CRS84") || strings.EqualFold(crs, "EPSG:4326") {
return "OGC:CRS84", nil
}
switch meta.CRSType {
- case geoarrow.CRSTypeSRID:
- return "srid:" + crs, nil
+ case geoarrow.CRSTypePROJJSON:
+ return "", errPROJJSONStringCRSNotSupported
case geoarrow.CRSTypeWKT22019:
- return "", errors.New("CRS type wkt2:2019 not supported")
+ return "", errWKT2CRSNotSupported
default:
- if len(crs) <= 32 {
- return crs, nil
- }
+ // Unset, AuthorityCode, SRID, and future CRSType values continue to
+ // the string-shape checks below.
+ }
- return "", errors.New("crs length too long")
+ if isWKT2CRSString(crs) {
+ return "", errWKT2CRSNotSupported
+ }
+ if meta.CRSType == geoarrow.CRSTypeSRID {
+ return "srid:" + crs, nil
}
+
+ // Bare or unrecognized string CRS values are opaque identifiers per the
+ // Iceberg type contract; preserve them for read/write compatibility.
+ return crs, nil
case checkCRSJSON(meta.CRS):
var crs map[string]json.RawMessage
@@ -1947,34 +2001,41 @@ func geoArrowCRSToIcebergCRS(meta geoarrow.Metadata) (string, error) {
idRaw, ok := crs["id"]
if !ok || len(idRaw) == 0 {
- return "", errors.New("unsupported CRS")
+ return "", errors.New("unsupported CRS: missing id")
}
var id map[string]json.RawMessage
if err := json.Unmarshal(idRaw, &id); err != nil {
- return "", errors.New("unsupported CRS")
+ return "", fmt.Errorf("unsupported CRS: invalid id: %w", err)
}
var authority string
- if err := json.Unmarshal(id["authority"], &authority); err != nil || authority == "" {
- return "", errors.New("unsupported CRS")
+ authorityRaw, ok := id["authority"]
+ if !ok || len(authorityRaw) == 0 {
+ return "", errors.New("unsupported CRS: missing id.authority")
+ }
+ if err := json.Unmarshal(authorityRaw, &authority); err != nil {
+ return "", fmt.Errorf("unsupported CRS: invalid id.authority: %w", err)
+ }
+ if authority == "" {
+ return "", errors.New("unsupported CRS: empty id.authority")
}
codeRaw := id["code"]
if len(codeRaw) == 0 {
- return "", errors.New("unsupported CRS")
+ return "", errors.New("unsupported CRS: missing id.code")
}
var code string
if err := json.Unmarshal(codeRaw, &code); err != nil {
var codeNum json.Number
if err := json.Unmarshal(codeRaw, &codeNum); err != nil {
- return "", errors.New("unsupported CRS")
+ return "", fmt.Errorf("unsupported CRS: invalid id.code: %w", err)
}
code = codeNum.String()
}
if code == "" {
- return "", errors.New("unsupported CRS")
+ return "", errors.New("unsupported CRS: empty id.code")
}
authorityCode := authority + ":" + code
diff --git a/table/arrow_utils_internal_test.go b/table/arrow_utils_internal_test.go
index c460c476..fbf8667c 100644
--- a/table/arrow_utils_internal_test.go
+++ b/table/arrow_utils_internal_test.go
@@ -20,6 +20,7 @@ package table
import (
"bytes"
"cmp"
+ "encoding/json"
"math"
"slices"
"testing"
@@ -491,3 +492,142 @@ func TestIcebergCRSToGeoArrowMetadata(t *testing.T) {
assert.Empty(t, meta.CRSType)
})
}
+
+func TestGeoArrowCRSToIcebergCRS(t *testing.T) {
+ stringCRS := func(s string) json.RawMessage {
+ raw, _ := json.Marshal(s) //nolint:errcheck // Marshalling a string can't fail
+
+ return raw
+ }
+
+ t.Run("accepts long authority code string", func(t *testing.T) {
+ const crs = "EPSGAuthorityLongName:CODE-12345678901234567890"
+
+ got, err := geoArrowCRSToIcebergCRS(geoarrow.Metadata{CRS: stringCRS(crs)})
+ require.NoError(t, err)
+ assert.Equal(t, crs, got)
+ })
+
+ t.Run("accepts arbitrary string CRS", func(t *testing.T) {
+ got, err := geoArrowCRSToIcebergCRS(geoarrow.Metadata{CRS: stringCRS("custom")})
+ require.NoError(t, err)
+ assert.Equal(t, "custom", got)
+ })
+
+ t.Run("canonical string CRS wins over type annotation", func(t *testing.T) {
+ for _, crsType := range []geoarrow.CRSType{
+ geoarrow.CRSTypeSRID,
+ geoarrow.CRSTypePROJJSON,
+ geoarrow.CRSTypeWKT22019,
+ } {
+ for _, crs := range []string{"EPSG:4326", "OGC:CRS84"} {
+ t.Run(string(crsType)+"/"+crs, func(t *testing.T) {
+ got, err := geoArrowCRSToIcebergCRS(geoarrow.Metadata{
+ CRS: stringCRS(crs),
+ CRSType: crsType,
+ })
+ require.NoError(t, err)
+ assert.Equal(t, "OGC:CRS84", got)
+ })
+ }
+ }
+ })
+
+ t.Run("numeric srid type annotation maps to srid", func(t *testing.T) {
+ got, err := geoArrowCRSToIcebergCRS(geoarrow.Metadata{
+ CRS: stringCRS("3857"),
+ CRSType: geoarrow.CRSTypeSRID,
+ })
+ require.NoError(t, err)
+ assert.Equal(t, "srid:3857", got)
+ })
+
+ t.Run("rejects empty string CRS", func(t *testing.T) {
+ _, err := geoArrowCRSToIcebergCRS(geoarrow.Metadata{CRS: stringCRS("")})
+ require.Error(t, err)
+ assert.ErrorContains(t, err, "unsupported CRS: empty string CRS")
+ })
+
+ t.Run("rejects projjson string CRS", func(t *testing.T) {
+ _, err := geoArrowCRSToIcebergCRS(geoarrow.Metadata{
+ CRS: stringCRS("EPSG:3857"),
+ CRSType: geoarrow.CRSTypePROJJSON,
+ })
+ require.Error(t, err)
+ assert.ErrorIs(t, err, errPROJJSONStringCRSNotSupported)
+ })
+
+ t.Run("bare WKT2 string uses not supported error", func(t *testing.T) {
+ for _, crs := range []string{
+ `GEOGCRS["WGS 84",DATUM["World Geodetic System 1984"]]`,
+ `COORDINATEMETADATA[SOURCECRS[GEOGCRS["WGS 84"]]]`,
+ `COORDINATEOPERATION["operation"]`,
+ `CRS["generic"]`,
+ } {
+ t.Run(crs, func(t *testing.T) {
+ _, err := geoArrowCRSToIcebergCRS(geoarrow.Metadata{
+ CRS: stringCRS(crs),
+ })
+ require.Error(t, err)
+ assert.ErrorIs(t, err, errWKT2CRSNotSupported)
+ })
+ }
+ })
+
+ t.Run("json object errors identify the failing member", func(t *testing.T) {
+ cases := []struct {
+ name string
+ rawCRS string
+ wantErr string
+ }{
+ {
+ name: "missing id",
+ rawCRS: `{"type":"GeographicCRS"}`,
+ wantErr: "unsupported CRS: missing id",
+ },
+ {
+ name: "invalid id",
+ rawCRS: `{"id":42}`,
+ wantErr: "unsupported CRS: invalid id",
+ },
+ {
+ name: "missing authority",
+ rawCRS: `{"id":{"code":4326}}`,
+ wantErr: "unsupported CRS: missing id.authority",
+ },
+ {
+ name: "invalid authority",
+ rawCRS: `{"id":{"authority":7,"code":4326}}`,
+ wantErr: "unsupported CRS: invalid id.authority",
+ },
+ {
+ name: "empty authority",
+ rawCRS: `{"id":{"authority":"","code":4326}}`,
+ wantErr: "unsupported CRS: empty id.authority",
+ },
+ {
+ name: "missing code",
+ rawCRS: `{"id":{"authority":"EPSG"}}`,
+ wantErr: "unsupported CRS: missing id.code",
+ },
+ {
+ name: "invalid code",
+ rawCRS: `{"id":{"authority":"EPSG","code":{}}}`,
+ wantErr: "unsupported CRS: invalid id.code",
+ },
+ {
+ name: "empty code",
+ rawCRS: `{"id":{"authority":"EPSG","code":""}}`,
+ wantErr: "unsupported CRS: empty id.code",
+ },
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ _, err := geoArrowCRSToIcebergCRS(geoarrow.Metadata{CRS: json.RawMessage(tc.rawCRS)})
+ require.Error(t, err)
+ assert.ErrorContains(t, err, tc.wantErr)
+ })
+ }
+ })
+}
diff --git a/table/arrow_utils_test.go b/table/arrow_utils_test.go
index 32d58363..2a3b380a 100644
--- a/table/arrow_utils_test.go
+++ b/table/arrow_utils_test.go
@@ -499,6 +499,8 @@ func TestIcebergGeoTypesToArrowSchema(t *testing.T) {
require.NoError(t, err)
geogEPSG4267, err := iceberg.GeographyTypeOf("EPSG:4267", "spherical")
require.NoError(t, err)
+ geomCustomCRS, err := iceberg.GeometryTypeOf("my-custom-crs")
+ require.NoError(t, err)
defaultGeometry, err := iceberg.GeometryTypeOf("OGC:CRS84")
require.NoError(t, err)
geomEPSG3857, err := iceberg.GeometryTypeOf("EPSG:3857")
@@ -530,6 +532,11 @@ func TestIcebergGeoTypesToArrowSchema(t *testing.T) {
ice: geomEPSG4267,
geoarrowMetaJSON: `{"crs":"EPSG:4267"}`,
},
+ {
+ name: "geometry_custom_string_crs",
+ ice: geomCustomCRS,
+ geoarrowMetaJSON: `{"crs":"my-custom-crs"}`,
+ },
// Geography with default CRS (default OGC:CRS84, spherical edges)
{
name: "geography_default_crs",
@@ -613,7 +620,12 @@ func TestIcebergGeoTypesToArrowSchema(t *testing.T) {
// EPSG:4326 is canonicalized before crs_type validation, so a mismatched type still maps to the default CRS.
name: "geometry_epsg_4326_incorrect_type",
ice: defaultGeometry,
- geoarrowMetaJSON: `{"crs":"epsg:4326", "crs_type":"projjson"}`,
+ geoarrowMetaJSON: `{"crs":"EPSG:4326","crs_type":"projjson"}`,
+ },
+ {
+ name: "geometry_epsg_4326_wkt2_type",
+ ice: defaultGeometry,
+ geoarrowMetaJSON: `{"crs":"EPSG:4326","crs_type":"wkt2:2019"}`,
},
// Translated from arrow-rs geo logical type read tests (https://github.com/apache/arrow-rs/pull/10065)
@@ -816,6 +828,14 @@ func TestIcebergGeoTypesToArrowSchema(t *testing.T) {
require.ErrorIs(t, err, iceberg.ErrInvalidSchema)
require.ErrorContains(t, err, "projjson CRS not supported yet")
+ stringArrowType, err := geoarrow.NewWKBType().Deserialize(arrow.BinaryTypes.Binary,
+ `{"crs_type":"projjson","crs":"EPSG:3857"}`)
+ require.NoError(t, err)
+
+ _, err = table.ArrowTypeToIceberg(stringArrowType, false)
+ require.Error(t, err)
+ require.ErrorContains(t, err, "CRS type projjson not supported for string CRS")
+
arrowType, err := geoarrow.NewWKBType().Deserialize(arrow.BinaryTypes.Binary,
`{"crs_type":"projjson","crs":{"id":{"authority":"OGC", "code":"CRS84"}}}`)
require.NoError(t, err)
@@ -840,7 +860,7 @@ func TestIcebergGeoTypesToArrowSchema(t *testing.T) {
_, err = table.ArrowTypeToIceberg(arrowTypeWithoutCRSType, false)
require.Error(t, err)
- require.ErrorContains(t, err, "crs length too long")
+ require.ErrorContains(t, err, "CRS type wkt2:2019 not supported")
})
t.Run("schema", func(t *testing.T) {