You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@iceberg.apache.org by "twuebi (via GitHub)" <gi...@apache.org> on 2025/09/08 16:06:29 UTC

[PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

twuebi opened a new pull request, #558:
URL: https://github.com/apache/iceberg-go/pull/558

   This PR:
   
   1. adds a funct-opts builder for PartitionSpec
   2. implements the remove PartitionSpec Update
   3. ports test relating to PartitionSpecs from iceberg-rust
   4. port missing validations from iceberg-rust to metadataBuilder
   
   
   It's mostly contained within partitions, there's a more fundamental change to json deserialization in there which initializes the deserialization target with `-1` in all int fields so that we are able to determine if we received / did not receive a certain field and are consequently able to reject requests where mandatory IDs are missing.


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "twuebi (via GitHub)" <gi...@apache.org>.
twuebi commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2334551905


##########
partitions.go:
##########
@@ -85,6 +87,130 @@ type PartitionSpec struct {
 	sourceIdToFields map[int][]PartitionField
 }
 
+type PartitionOption func(*PartitionSpec) error
+
+func NewPartitionSpecOpts(opts ...PartitionOption) (PartitionSpec, error) {
+	spec := PartitionSpec{
+		id: 0,
+	}
+	for _, opt := range opts {
+		if err := opt(&spec); err != nil {
+			return PartitionSpec{}, err
+		}
+	}
+	spec.initialize()
+
+	return spec, nil
+}
+
+func WithSpecID(id int) PartitionOption {
+	return func(p *PartitionSpec) error {
+		p.id = id
+
+		return nil
+	}
+}
+
+func AddPartitionFieldBySourceID(sourceID int, targetName string, transform Transform, schema *Schema, fieldID *int) PartitionOption {
+	return func(p *PartitionSpec) error {
+		if schema == nil {
+			return errors.New("cannot add partition field with nil schema")
+		}
+		field, ok := schema.FindFieldByID(sourceID)
+		if !ok {
+			return fmt.Errorf("cannot find source column with id: %d in schema", sourceID)
+		}
+		err := addSpecFieldInternal(p, targetName, field, transform, fieldID)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	}
+}
+
+func addSpecFieldInternal(p *PartitionSpec, targetName string, field NestedField, transform Transform, fieldID *int) error {
+	if targetName == "" {
+		return errors.New("cannot use empty partition name")
+	}
+	for _, existingField := range p.fields {
+		if existingField.Name == targetName {
+			return errors.New("duplicate partition name: " + targetName)
+		}
+	}
+	var fieldIDValue int
+	if fieldID == nil {
+		fieldIDValue = unassignedFieldID
+	} else {
+		fieldIDValue = *fieldID
+	}
+	if err := p.checkForRedundantPartitions(field.ID, transform); err != nil {
+		return err
+	}
+	unboundField := PartitionField{
+		SourceID:  field.ID,
+		FieldID:   fieldIDValue,
+		Name:      targetName,
+		Transform: transform,
+	}
+	p.fields = append(p.fields, unboundField)
+
+	return nil
+}
+
+func (p *PartitionSpec) checkForRedundantPartitions(sourceID int, transform Transform) error {
+	for _, field := range p.fields {
+		if field.SourceID == sourceID && field.Transform.String() == transform.String() {
+			return fmt.Errorf("cannot add redundant partition with source id %d and transform %s. A partition with the same source id and transform already exists with name: %s", sourceID, transform, field.Name)
+		}
+	}
+
+	return nil
+}
+
+func (p *PartitionSpec) SetID(id int) {
+	p.id = id
+}
+
+func (ps *PartitionSpec) AssignPartitionFieldIds(lastAssignedFieldIDPtr *int) error {
+	// This is set_field_ids from iceberg-rust
+	// Already assigned partition ids. If we see one of these during iteration,
+	// we skip it.
+	assignedIds := make(map[int]struct{})
+	for _, field := range ps.fields {
+		if field.FieldID != unassignedFieldID {
+			if _, exists := assignedIds[field.FieldID]; exists {
+				return fmt.Errorf("duplicate field ID provided: %d", field.FieldID)
+			}
+			assignedIds[field.FieldID] = struct{}{}
+		}
+	}
+
+	lastAssignedFieldID := ps.LastAssignedFieldID()
+	if lastAssignedFieldIDPtr != nil {
+		lastAssignedFieldID = *lastAssignedFieldIDPtr
+	}

Review Comment:
   I went for the pointer since it's explicit that it's either set or unset without resort to sentinel values, if sentinel values is more idiomatic, I can do that too.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "twuebi (via GitHub)" <gi...@apache.org>.
twuebi commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2334555595


##########
partitions.go:
##########
@@ -85,6 +87,130 @@ type PartitionSpec struct {
 	sourceIdToFields map[int][]PartitionField
 }
 
+type PartitionOption func(*PartitionSpec) error
+
+func NewPartitionSpecOpts(opts ...PartitionOption) (PartitionSpec, error) {
+	spec := PartitionSpec{
+		id: 0,
+	}
+	for _, opt := range opts {
+		if err := opt(&spec); err != nil {
+			return PartitionSpec{}, err
+		}
+	}
+	spec.initialize()
+
+	return spec, nil
+}
+
+func WithSpecID(id int) PartitionOption {
+	return func(p *PartitionSpec) error {
+		p.id = id
+
+		return nil
+	}
+}
+
+func AddPartitionFieldBySourceID(sourceID int, targetName string, transform Transform, schema *Schema, fieldID *int) PartitionOption {
+	return func(p *PartitionSpec) error {
+		if schema == nil {
+			return errors.New("cannot add partition field with nil schema")
+		}
+		field, ok := schema.FindFieldByID(sourceID)
+		if !ok {
+			return fmt.Errorf("cannot find source column with id: %d in schema", sourceID)
+		}
+		err := addSpecFieldInternal(p, targetName, field, transform, fieldID)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	}
+}
+
+func addSpecFieldInternal(p *PartitionSpec, targetName string, field NestedField, transform Transform, fieldID *int) error {
+	if targetName == "" {
+		return errors.New("cannot use empty partition name")
+	}
+	for _, existingField := range p.fields {
+		if existingField.Name == targetName {
+			return errors.New("duplicate partition name: " + targetName)
+		}
+	}
+	var fieldIDValue int
+	if fieldID == nil {
+		fieldIDValue = unassignedFieldID
+	} else {
+		fieldIDValue = *fieldID
+	}
+	if err := p.checkForRedundantPartitions(field.ID, transform); err != nil {
+		return err
+	}
+	unboundField := PartitionField{
+		SourceID:  field.ID,
+		FieldID:   fieldIDValue,
+		Name:      targetName,
+		Transform: transform,
+	}
+	p.fields = append(p.fields, unboundField)
+
+	return nil
+}
+
+func (p *PartitionSpec) checkForRedundantPartitions(sourceID int, transform Transform) error {
+	for _, field := range p.fields {
+		if field.SourceID == sourceID && field.Transform.String() == transform.String() {
+			return fmt.Errorf("cannot add redundant partition with source id %d and transform %s. A partition with the same source id and transform already exists with name: %s", sourceID, transform, field.Name)
+		}
+	}
+
+	return nil
+}
+
+func (p *PartitionSpec) SetID(id int) {
+	p.id = id
+}

Review Comment:
   removed, only callsite was better served by moving all of it into the new method `PartitionSpec.BindToSchema`



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "twuebi (via GitHub)" <gi...@apache.org>.
twuebi commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2337006671


##########
catalog/rest/rest.go:
##########
@@ -750,27 +750,12 @@ func (r *Catalog) CreateTable(ctx context.Context, identifier table.Identifier,
 		o(&cfg)
 	}
 
-	freshSchema, err := iceberg.AssignFreshSchemaIDs(schema, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	freshPartitionSpec, err := iceberg.AssignFreshPartitionSpecIDs(cfg.PartitionSpec, schema, freshSchema)
-	if err != nil {
-		return nil, err
-	}
-
-	freshSortOrder, err := table.AssignFreshSortOrderIDs(cfg.SortOrder, schema, freshSchema)
-	if err != nil {
-		return nil, err
-	}

Review Comment:
   neither java, nor rust are doing this, I'd prefer to get rid of it to remove the last callsite of `iceberg.AssignFreshPartitionSpecIDs`. Any rest catalog needs to recheck this on createTable, servers can't trust clients to do this beforehand so I think this is largely redundant



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "twuebi (via GitHub)" <gi...@apache.org>.
twuebi commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2331208797


##########
partitions.go:
##########
@@ -85,6 +87,130 @@ type PartitionSpec struct {
 	sourceIdToFields map[int][]PartitionField
 }
 
+type PartitionOption func(*PartitionSpec) error
+
+func NewPartitionSpecOpts(opts ...PartitionOption) (PartitionSpec, error) {

Review Comment:
   I think it'd make sense to eventually get rid of `NewPartitionSpec` since it allows to construct invalid partition specs?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "twuebi (via GitHub)" <gi...@apache.org>.
twuebi commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2334547108


##########
table/updates.go:
##########
@@ -553,20 +553,22 @@ func (u *removeSnapshotRefUpdate) Apply(builder *MetadataBuilder) error {
 
 type removeSpecUpdate struct {
 	baseUpdate
-	SpecIds []int64 `json:"spec-ids"`
+	SpecIds []int `json:"spec-ids"`

Review Comment:
   java uses `int` which is signed 32bit, rust uses `i32`, I think for cross language compatibility's sake it makes sense to keep it 32bit



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "twuebi (via GitHub)" <gi...@apache.org>.
twuebi commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2356598102


##########
catalog/rest/rest.go:
##########
@@ -750,27 +750,12 @@ func (r *Catalog) CreateTable(ctx context.Context, identifier table.Identifier,
 		o(&cfg)
 	}
 
-	freshSchema, err := iceberg.AssignFreshSchemaIDs(schema, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	freshPartitionSpec, err := iceberg.AssignFreshPartitionSpecIDs(cfg.PartitionSpec, schema, freshSchema)
-	if err != nil {
-		return nil, err
-	}
-
-	freshSortOrder, err := table.AssignFreshSortOrderIDs(cfg.SortOrder, schema, freshSchema)
-	if err != nil {
-		return nil, err
-	}

Review Comment:
   Not sure how much of a go-ism that'd be - I thought it'd make sense to make those fields private so that mutators would protect users from building stuff that won't work? 
   
   That'd somewhat conflict with funct-opts being consumer-extendable I guess?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2331152651


##########
partitions.go:
##########
@@ -30,6 +31,7 @@ import (
 const (
 	PartitionDataIDStart   = 1000
 	InitialPartitionSpecID = 0
+	unassignedFieldID      = 0

Review Comment:
   should this be -1 instead of 0?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "twuebi (via GitHub)" <gi...@apache.org>.
twuebi commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2331227620


##########
table/updates.go:
##########
@@ -553,20 +553,22 @@ func (u *removeSnapshotRefUpdate) Apply(builder *MetadataBuilder) error {
 
 type removeSpecUpdate struct {
 	baseUpdate
-	SpecIds []int64 `json:"spec-ids"`
+	SpecIds []int `json:"spec-ids"`

Review Comment:
   specIds are int not int64, I made a mistake when originally adding the updates



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "twuebi (via GitHub)" <gi...@apache.org>.
twuebi commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2334611475


##########
table/updates.go:
##########
@@ -553,20 +553,22 @@ func (u *removeSnapshotRefUpdate) Apply(builder *MetadataBuilder) error {
 
 type removeSpecUpdate struct {
 	baseUpdate
-	SpecIds []int64 `json:"spec-ids"`
+	SpecIds []int `json:"spec-ids"`

Review Comment:
   https://github.com/apache/iceberg-go/issues/559



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2334903269


##########
partitions.go:
##########
@@ -85,6 +87,130 @@ type PartitionSpec struct {
 	sourceIdToFields map[int][]PartitionField
 }
 
+type PartitionOption func(*PartitionSpec) error
+
+func NewPartitionSpecOpts(opts ...PartitionOption) (PartitionSpec, error) {
+	spec := PartitionSpec{
+		id: 0,
+	}
+	for _, opt := range opts {
+		if err := opt(&spec); err != nil {
+			return PartitionSpec{}, err
+		}
+	}
+	spec.initialize()
+
+	return spec, nil
+}
+
+func WithSpecID(id int) PartitionOption {
+	return func(p *PartitionSpec) error {
+		p.id = id
+
+		return nil
+	}
+}
+
+func AddPartitionFieldBySourceID(sourceID int, targetName string, transform Transform, schema *Schema, fieldID *int) PartitionOption {
+	return func(p *PartitionSpec) error {
+		if schema == nil {
+			return errors.New("cannot add partition field with nil schema")
+		}
+		field, ok := schema.FindFieldByID(sourceID)
+		if !ok {
+			return fmt.Errorf("cannot find source column with id: %d in schema", sourceID)
+		}
+		err := addSpecFieldInternal(p, targetName, field, transform, fieldID)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	}
+}
+
+func addSpecFieldInternal(p *PartitionSpec, targetName string, field NestedField, transform Transform, fieldID *int) error {
+	if targetName == "" {
+		return errors.New("cannot use empty partition name")
+	}
+	for _, existingField := range p.fields {
+		if existingField.Name == targetName {
+			return errors.New("duplicate partition name: " + targetName)
+		}
+	}
+	var fieldIDValue int
+	if fieldID == nil {
+		fieldIDValue = unassignedFieldID
+	} else {
+		fieldIDValue = *fieldID
+	}
+	if err := p.checkForRedundantPartitions(field.ID, transform); err != nil {
+		return err
+	}
+	unboundField := PartitionField{
+		SourceID:  field.ID,
+		FieldID:   fieldIDValue,
+		Name:      targetName,
+		Transform: transform,
+	}
+	p.fields = append(p.fields, unboundField)
+
+	return nil
+}
+
+func (p *PartitionSpec) checkForRedundantPartitions(sourceID int, transform Transform) error {
+	for _, field := range p.fields {
+		if field.SourceID == sourceID && field.Transform.String() == transform.String() {
+			return fmt.Errorf("cannot add redundant partition with source id %d and transform %s. A partition with the same source id and transform already exists with name: %s", sourceID, transform, field.Name)
+		}
+	}
+
+	return nil
+}
+
+func (p *PartitionSpec) SetID(id int) {
+	p.id = id
+}
+
+func (ps *PartitionSpec) AssignPartitionFieldIds(lastAssignedFieldIDPtr *int) error {

Review Comment:
   I'd be fine with that



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade commented on PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#issuecomment-3303753011

   Hmm, i wonder why the REST catalog integration tests are failing. It's likely somehow related to this change as they don't seem to be failing on main.


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "twuebi (via GitHub)" <gi...@apache.org>.
twuebi commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2356419582


##########
catalog/rest/rest.go:
##########
@@ -750,27 +750,12 @@ func (r *Catalog) CreateTable(ctx context.Context, identifier table.Identifier,
 		o(&cfg)
 	}
 
-	freshSchema, err := iceberg.AssignFreshSchemaIDs(schema, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	freshPartitionSpec, err := iceberg.AssignFreshPartitionSpecIDs(cfg.PartitionSpec, schema, freshSchema)
-	if err != nil {
-		return nil, err
-	}
-
-	freshSortOrder, err := table.AssignFreshSortOrderIDs(cfg.SortOrder, schema, freshSchema)
-	if err != nil {
-		return nil, err
-	}

Review Comment:
   Will take a look



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "twuebi (via GitHub)" <gi...@apache.org>.
twuebi commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2356523363


##########
catalog/rest/rest.go:
##########
@@ -750,27 +750,12 @@ func (r *Catalog) CreateTable(ctx context.Context, identifier table.Identifier,
 		o(&cfg)
 	}
 
-	freshSchema, err := iceberg.AssignFreshSchemaIDs(schema, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	freshPartitionSpec, err := iceberg.AssignFreshPartitionSpecIDs(cfg.PartitionSpec, schema, freshSchema)
-	if err != nil {
-		return nil, err
-	}
-
-	freshSortOrder, err := table.AssignFreshSortOrderIDs(cfg.SortOrder, schema, freshSchema)
-	if err != nil {
-		return nil, err
-	}

Review Comment:
   will investigate a bit further, worst case, I'll just put the reassignments back in



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "twuebi (via GitHub)" <gi...@apache.org>.
twuebi commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2356550711


##########
catalog/rest/rest.go:
##########
@@ -750,27 +750,12 @@ func (r *Catalog) CreateTable(ctx context.Context, identifier table.Identifier,
 		o(&cfg)
 	}
 
-	freshSchema, err := iceberg.AssignFreshSchemaIDs(schema, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	freshPartitionSpec, err := iceberg.AssignFreshPartitionSpecIDs(cfg.PartitionSpec, schema, freshSchema)
-	if err != nil {
-		return nil, err
-	}
-
-	freshSortOrder, err := table.AssignFreshSortOrderIDs(cfg.SortOrder, schema, freshSchema)
-	if err != nil {
-		return nil, err
-	}

Review Comment:
   `[qtp2012993836-28] WARN org.eclipse.jetty.server.HttpChannel - handleException /v1/namespaces/TEST%20NS/tables com.fasterxml.jackson.databind.JsonMappingException: Cannot parse missing field: fields (through reference chain: org.apache.iceberg.rest.requests.CreateTableRequest["write-order"])`
   
   yeah, also just saw this



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "twuebi (via GitHub)" <gi...@apache.org>.
twuebi commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2334607801


##########
table/updates.go:
##########
@@ -553,20 +553,22 @@ func (u *removeSnapshotRefUpdate) Apply(builder *MetadataBuilder) error {
 
 type removeSpecUpdate struct {
 	baseUpdate
-	SpecIds []int64 `json:"spec-ids"`
+	SpecIds []int `json:"spec-ids"`

Review Comment:
   fixing that now will have a lot of changes all over the place, I'll create an issue and we'll do it as a follow up?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "twuebi (via GitHub)" <gi...@apache.org>.
twuebi commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2356499895


##########
catalog/rest/rest.go:
##########
@@ -750,27 +750,12 @@ func (r *Catalog) CreateTable(ctx context.Context, identifier table.Identifier,
 		o(&cfg)
 	}
 
-	freshSchema, err := iceberg.AssignFreshSchemaIDs(schema, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	freshPartitionSpec, err := iceberg.AssignFreshPartitionSpecIDs(cfg.PartitionSpec, schema, freshSchema)
-	if err != nil {
-		return nil, err
-	}
-
-	freshSortOrder, err := table.AssignFreshSortOrderIDs(cfg.SortOrder, schema, freshSchema)
-	if err != nil {
-		return nil, err
-	}

Review Comment:
   It makes very little sense for view or namespace changes to fail based on these changes?
   
   ```
       --- PASS: TestRestIntegration/TestCreateNamespace (0.03s)
       --- FAIL: TestRestIntegration/TestCreateTable (0.02s)
       --- FAIL: TestRestIntegration/TestCreateView (0.01s)
       --- PASS: TestRestIntegration/TestLoadNamespaceProps (0.01s)
       --- PASS: TestRestIntegration/TestUpdateNamespaceProps (0.03s)
       --- FAIL: TestRestIntegration/TestWriteCommitTable (0.02s)
   ```
   
   maybe test setup is already failing? 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "twuebi (via GitHub)" <gi...@apache.org>.
twuebi commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2356525973


##########
catalog/rest/rest.go:
##########
@@ -750,27 +750,12 @@ func (r *Catalog) CreateTable(ctx context.Context, identifier table.Identifier,
 		o(&cfg)
 	}
 
-	freshSchema, err := iceberg.AssignFreshSchemaIDs(schema, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	freshPartitionSpec, err := iceberg.AssignFreshPartitionSpecIDs(cfg.PartitionSpec, schema, freshSchema)
-	if err != nil {
-		return nil, err
-	}
-
-	freshSortOrder, err := table.AssignFreshSortOrderIDs(cfg.SortOrder, schema, freshSchema)
-	if err != nil {
-		return nil, err
-	}

Review Comment:
   ok, failing view makes sense, it starts by creating a view



##########
catalog/rest/rest.go:
##########
@@ -750,27 +750,12 @@ func (r *Catalog) CreateTable(ctx context.Context, identifier table.Identifier,
 		o(&cfg)
 	}
 
-	freshSchema, err := iceberg.AssignFreshSchemaIDs(schema, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	freshPartitionSpec, err := iceberg.AssignFreshPartitionSpecIDs(cfg.PartitionSpec, schema, freshSchema)
-	if err != nil {
-		return nil, err
-	}
-
-	freshSortOrder, err := table.AssignFreshSortOrderIDs(cfg.SortOrder, schema, freshSchema)
-	if err != nil {
-		return nil, err
-	}

Review Comment:
   ok, failing view makes sense, it starts by creating a table



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade merged PR #558:
URL: https://github.com/apache/iceberg-go/pull/558


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2331295950


##########
table/updates.go:
##########
@@ -553,20 +553,22 @@ func (u *removeSnapshotRefUpdate) Apply(builder *MetadataBuilder) error {
 
 type removeSpecUpdate struct {
 	baseUpdate
-	SpecIds []int64 `json:"spec-ids"`
+	SpecIds []int `json:"spec-ids"`

Review Comment:
   technically spec ids are `JSON int`s according to the spec, which means that an int64 would be valid. But I'm fine with changing them back to just being an `int` for now



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "twuebi (via GitHub)" <gi...@apache.org>.
twuebi commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2334001345


##########
partitions.go:
##########
@@ -85,6 +87,130 @@ type PartitionSpec struct {
 	sourceIdToFields map[int][]PartitionField
 }
 
+type PartitionOption func(*PartitionSpec) error
+
+func NewPartitionSpecOpts(opts ...PartitionOption) (PartitionSpec, error) {
+	spec := PartitionSpec{
+		id: 0,
+	}
+	for _, opt := range opts {
+		if err := opt(&spec); err != nil {
+			return PartitionSpec{}, err
+		}
+	}
+	spec.initialize()
+
+	return spec, nil
+}
+
+func WithSpecID(id int) PartitionOption {
+	return func(p *PartitionSpec) error {
+		p.id = id
+
+		return nil
+	}
+}
+
+func AddPartitionFieldBySourceID(sourceID int, targetName string, transform Transform, schema *Schema, fieldID *int) PartitionOption {
+	return func(p *PartitionSpec) error {
+		if schema == nil {
+			return errors.New("cannot add partition field with nil schema")
+		}
+		field, ok := schema.FindFieldByID(sourceID)
+		if !ok {
+			return fmt.Errorf("cannot find source column with id: %d in schema", sourceID)
+		}
+		err := addSpecFieldInternal(p, targetName, field, transform, fieldID)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	}
+}
+
+func addSpecFieldInternal(p *PartitionSpec, targetName string, field NestedField, transform Transform, fieldID *int) error {
+	if targetName == "" {
+		return errors.New("cannot use empty partition name")
+	}
+	for _, existingField := range p.fields {
+		if existingField.Name == targetName {
+			return errors.New("duplicate partition name: " + targetName)
+		}
+	}
+	var fieldIDValue int
+	if fieldID == nil {
+		fieldIDValue = unassignedFieldID
+	} else {
+		fieldIDValue = *fieldID
+	}
+	if err := p.checkForRedundantPartitions(field.ID, transform); err != nil {
+		return err
+	}
+	unboundField := PartitionField{
+		SourceID:  field.ID,
+		FieldID:   fieldIDValue,
+		Name:      targetName,
+		Transform: transform,
+	}
+	p.fields = append(p.fields, unboundField)
+
+	return nil
+}
+
+func (p *PartitionSpec) checkForRedundantPartitions(sourceID int, transform Transform) error {
+	for _, field := range p.fields {
+		if field.SourceID == sourceID && field.Transform.String() == transform.String() {
+			return fmt.Errorf("cannot add redundant partition with source id %d and transform %s. A partition with the same source id and transform already exists with name: %s", sourceID, transform, field.Name)
+		}
+	}
+
+	return nil
+}
+
+func (p *PartitionSpec) SetID(id int) {
+	p.id = id
+}
+
+func (ps *PartitionSpec) AssignPartitionFieldIds(lastAssignedFieldIDPtr *int) error {
+	// This is set_field_ids from iceberg-rust
+	// Already assigned partition ids. If we see one of these during iteration,
+	// we skip it.
+	assignedIds := make(map[int]struct{})
+	for _, field := range ps.fields {
+		if field.FieldID != unassignedFieldID {
+			if _, exists := assignedIds[field.FieldID]; exists {
+				return fmt.Errorf("duplicate field ID provided: %d", field.FieldID)
+			}
+			assignedIds[field.FieldID] = struct{}{}
+		}
+	}

Review Comment:
   I thought it'd be best to have it called always from addPartitionSpec to make sure it's always run, putting it into the constructors / initialize of PartitionSpec would also work as long as the fields remain unexported, we make sure to call it on unmarshal too & we won't be adding any AddField function in the future.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "twuebi (via GitHub)" <gi...@apache.org>.
twuebi commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2334001345


##########
partitions.go:
##########
@@ -85,6 +87,130 @@ type PartitionSpec struct {
 	sourceIdToFields map[int][]PartitionField
 }
 
+type PartitionOption func(*PartitionSpec) error
+
+func NewPartitionSpecOpts(opts ...PartitionOption) (PartitionSpec, error) {
+	spec := PartitionSpec{
+		id: 0,
+	}
+	for _, opt := range opts {
+		if err := opt(&spec); err != nil {
+			return PartitionSpec{}, err
+		}
+	}
+	spec.initialize()
+
+	return spec, nil
+}
+
+func WithSpecID(id int) PartitionOption {
+	return func(p *PartitionSpec) error {
+		p.id = id
+
+		return nil
+	}
+}
+
+func AddPartitionFieldBySourceID(sourceID int, targetName string, transform Transform, schema *Schema, fieldID *int) PartitionOption {
+	return func(p *PartitionSpec) error {
+		if schema == nil {
+			return errors.New("cannot add partition field with nil schema")
+		}
+		field, ok := schema.FindFieldByID(sourceID)
+		if !ok {
+			return fmt.Errorf("cannot find source column with id: %d in schema", sourceID)
+		}
+		err := addSpecFieldInternal(p, targetName, field, transform, fieldID)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	}
+}
+
+func addSpecFieldInternal(p *PartitionSpec, targetName string, field NestedField, transform Transform, fieldID *int) error {
+	if targetName == "" {
+		return errors.New("cannot use empty partition name")
+	}
+	for _, existingField := range p.fields {
+		if existingField.Name == targetName {
+			return errors.New("duplicate partition name: " + targetName)
+		}
+	}
+	var fieldIDValue int
+	if fieldID == nil {
+		fieldIDValue = unassignedFieldID
+	} else {
+		fieldIDValue = *fieldID
+	}
+	if err := p.checkForRedundantPartitions(field.ID, transform); err != nil {
+		return err
+	}
+	unboundField := PartitionField{
+		SourceID:  field.ID,
+		FieldID:   fieldIDValue,
+		Name:      targetName,
+		Transform: transform,
+	}
+	p.fields = append(p.fields, unboundField)
+
+	return nil
+}
+
+func (p *PartitionSpec) checkForRedundantPartitions(sourceID int, transform Transform) error {
+	for _, field := range p.fields {
+		if field.SourceID == sourceID && field.Transform.String() == transform.String() {
+			return fmt.Errorf("cannot add redundant partition with source id %d and transform %s. A partition with the same source id and transform already exists with name: %s", sourceID, transform, field.Name)
+		}
+	}
+
+	return nil
+}
+
+func (p *PartitionSpec) SetID(id int) {
+	p.id = id
+}
+
+func (ps *PartitionSpec) AssignPartitionFieldIds(lastAssignedFieldIDPtr *int) error {
+	// This is set_field_ids from iceberg-rust
+	// Already assigned partition ids. If we see one of these during iteration,
+	// we skip it.
+	assignedIds := make(map[int]struct{})
+	for _, field := range ps.fields {
+		if field.FieldID != unassignedFieldID {
+			if _, exists := assignedIds[field.FieldID]; exists {
+				return fmt.Errorf("duplicate field ID provided: %d", field.FieldID)
+			}
+			assignedIds[field.FieldID] = struct{}{}
+		}
+	}

Review Comment:
   I thought it'd be best to have it called always from addPartitionSpec to make sure it's always run, putting it into the constructors / initialize of PartitionSpec would also work as long as the fields remain unexported & we make sure to call it on unmarshal too 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "twuebi (via GitHub)" <gi...@apache.org>.
twuebi commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2331232751


##########
table/metadata.go:
##########
@@ -134,6 +134,8 @@ type Metadata interface {
 	NameMapping() iceberg.NameMapping
 
 	LastSequenceNumber() int64
+
+	FormatVersion() int

Review Comment:
   didn't realize that one existed, will change



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2356544285


##########
catalog/rest/rest.go:
##########
@@ -750,27 +750,12 @@ func (r *Catalog) CreateTable(ctx context.Context, identifier table.Identifier,
 		o(&cfg)
 	}
 
-	freshSchema, err := iceberg.AssignFreshSchemaIDs(schema, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	freshPartitionSpec, err := iceberg.AssignFreshPartitionSpecIDs(cfg.PartitionSpec, schema, freshSchema)
-	if err != nil {
-		return nil, err
-	}
-
-	freshSortOrder, err := table.AssignFreshSortOrderIDs(cfg.SortOrder, schema, freshSchema)
-	if err != nil {
-		return nil, err
-	}

Review Comment:
   Looks like the failure is caused by `"write-order":{"order-id": 0, "fields": null}` instead of `"fields": []`. So something about the json marshalling of the payload without the Assign functions is leaving a slice as `null` instead of an empty slice. Let me take a look



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "twuebi (via GitHub)" <gi...@apache.org>.
twuebi commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2356563617


##########
catalog/rest/rest.go:
##########
@@ -750,27 +750,12 @@ func (r *Catalog) CreateTable(ctx context.Context, identifier table.Identifier,
 		o(&cfg)
 	}
 
-	freshSchema, err := iceberg.AssignFreshSchemaIDs(schema, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	freshPartitionSpec, err := iceberg.AssignFreshPartitionSpecIDs(cfg.PartitionSpec, schema, freshSchema)
-	if err != nil {
-		return nil, err
-	}
-
-	freshSortOrder, err := table.AssignFreshSortOrderIDs(cfg.SortOrder, schema, freshSchema)
-	if err != nil {
-		return nil, err
-	}

Review Comment:
   Yeah, it's just the null value of `var cfg catalog.CreateTableCfg` ruining the day



##########
catalog/rest/rest.go:
##########
@@ -750,27 +750,12 @@ func (r *Catalog) CreateTable(ctx context.Context, identifier table.Identifier,
 		o(&cfg)
 	}
 
-	freshSchema, err := iceberg.AssignFreshSchemaIDs(schema, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	freshPartitionSpec, err := iceberg.AssignFreshPartitionSpecIDs(cfg.PartitionSpec, schema, freshSchema)
-	if err != nil {
-		return nil, err
-	}
-
-	freshSortOrder, err := table.AssignFreshSortOrderIDs(cfg.SortOrder, schema, freshSchema)
-	if err != nil {
-		return nil, err
-	}

Review Comment:
   Yeah, it's just the zero value of `var cfg catalog.CreateTableCfg` ruining the day



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2356576547


##########
catalog/rest/rest.go:
##########
@@ -750,27 +750,12 @@ func (r *Catalog) CreateTable(ctx context.Context, identifier table.Identifier,
 		o(&cfg)
 	}
 
-	freshSchema, err := iceberg.AssignFreshSchemaIDs(schema, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	freshPartitionSpec, err := iceberg.AssignFreshPartitionSpecIDs(cfg.PartitionSpec, schema, freshSchema)
-	if err != nil {
-		return nil, err
-	}
-
-	freshSortOrder, err := table.AssignFreshSortOrderIDs(cfg.SortOrder, schema, freshSchema)
-	if err != nil {
-		return nil, err
-	}

Review Comment:
   yea, the default in `CreateTableCfg` is the zero value which has `nil` for the slice instead of an empty slice. The rest-fixture errors on the `null`.
   
   You can fix it by adding:
   
   ```go
   if cfg.SortOrder.Fields == nil {
       cfg.SortOrder = table.UnsortedSortOrder
   }
   ```
   
   before the payload is created



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "twuebi (via GitHub)" <gi...@apache.org>.
twuebi commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2356590662


##########
catalog/rest/rest.go:
##########
@@ -750,27 +750,12 @@ func (r *Catalog) CreateTable(ctx context.Context, identifier table.Identifier,
 		o(&cfg)
 	}
 
-	freshSchema, err := iceberg.AssignFreshSchemaIDs(schema, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	freshPartitionSpec, err := iceberg.AssignFreshPartitionSpecIDs(cfg.PartitionSpec, schema, freshSchema)
-	if err != nil {
-		return nil, err
-	}
-
-	freshSortOrder, err := table.AssignFreshSortOrderIDs(cfg.SortOrder, schema, freshSchema)
-	if err != nil {
-		return nil, err
-	}

Review Comment:
   I was about to add a constructor which would put sane defaults?
   
   
   ```
   func NewCreateTableCfg() CreateTableCfg {
   	return CreateTableCfg{
   		 Location:      "",
   		 PartitionSpec: nil,
   		 SortOrder:     table.UnsortedSortOrder,
   		 Properties:    nil,
   	}
   }
   ```
   
   



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2356514762


##########
catalog/rest/rest.go:
##########
@@ -750,27 +750,12 @@ func (r *Catalog) CreateTable(ctx context.Context, identifier table.Identifier,
 		o(&cfg)
 	}
 
-	freshSchema, err := iceberg.AssignFreshSchemaIDs(schema, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	freshPartitionSpec, err := iceberg.AssignFreshPartitionSpecIDs(cfg.PartitionSpec, schema, freshSchema)
-	if err != nil {
-		return nil, err
-	}
-
-	freshSortOrder, err := table.AssignFreshSortOrderIDs(cfg.SortOrder, schema, freshSchema)
-	if err != nil {
-		return nil, err
-	}

Review Comment:
   Yea, just confirmed that current main passes without issue. Most likely the fact that we're not re-assigning the field/partition/schema IDs is getting rejected by the iceberg rest image



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "twuebi (via GitHub)" <gi...@apache.org>.
twuebi commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2337009926


##########
table/metadata.go:
##########
@@ -1278,63 +1384,117 @@ func NewMetadata(sc *iceberg.Schema, partitions *iceberg.PartitionSpec, sortOrde
 
 // NewMetadataWithUUID is like NewMetadata, but allows the caller to specify the UUID of the table rather than creating a new one.
 func NewMetadataWithUUID(sc *iceberg.Schema, partitions *iceberg.PartitionSpec, sortOrder SortOrder, location string, props iceberg.Properties, tableUuid uuid.UUID) (Metadata, error) {
-	freshSchema, err := iceberg.AssignFreshSchemaIDs(sc, nil)
+	if tableUuid == uuid.Nil {
+		tableUuid = uuid.New()
+	}
+	var err error
+	formatVersion := DefaultFormatVersion
+	if props != nil {
+		verStr, ok := props["format-version"]
+		if ok {
+			if formatVersion, err = strconv.Atoi(verStr); err != nil {
+				formatVersion = DefaultFormatVersion
+			}
+			delete(props, "format-version")
+		}
+	}
+
+	reassignedIds, err := reassignIDs(sc, partitions, sortOrder)
 	if err != nil {
 		return nil, err
 	}
 
-	freshPartitions, err := iceberg.AssignFreshPartitionSpecIDs(partitions, sc, freshSchema)
+	builder, err := NewMetadataBuilder()
 	if err != nil {
 		return nil, err
 	}
 
-	freshSortOrder, err := AssignFreshSortOrderIDs(sortOrder, sc, freshSchema)
+	_, err = builder.SetFormatVersion(formatVersion)
 	if err != nil {
 		return nil, err
 	}
 
-	if tableUuid == uuid.Nil {
-		tableUuid = uuid.New()
+	_, err = builder.SetUUID(tableUuid)
+	if err != nil {
+		return nil, err
 	}
 
-	formatVersion := DefaultFormatVersion
-	if props != nil {
-		verStr, ok := props["format-version"]
-		if ok {
-			if formatVersion, err = strconv.Atoi(verStr); err != nil {
-				formatVersion = DefaultFormatVersion
-			}
-			delete(props, "format-version")
+	_, err = builder.AddSortOrder(&reassignedIds.sortOrder, true)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = builder.SetDefaultSortOrderID(-1)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = builder.AddSchema(reassignedIds.schema)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = builder.SetCurrentSchemaID(-1)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = builder.AddPartitionSpec(reassignedIds.partitionSpec, true)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = builder.SetLoc(location)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = builder.SetProperties(props)
+	if err != nil {
+		return nil, err
+	}

Review Comment:
   We should make this a funct-opts thing eventually, I think, I went for the builder here so that there's a single place of validation



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "twuebi (via GitHub)" <gi...@apache.org>.
twuebi commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2333992081


##########
partitions.go:
##########
@@ -85,6 +87,130 @@ type PartitionSpec struct {
 	sourceIdToFields map[int][]PartitionField
 }
 
+type PartitionOption func(*PartitionSpec) error
+
+func NewPartitionSpecOpts(opts ...PartitionOption) (PartitionSpec, error) {
+	spec := PartitionSpec{
+		id: 0,
+	}
+	for _, opt := range opts {
+		if err := opt(&spec); err != nil {
+			return PartitionSpec{}, err
+		}
+	}
+	spec.initialize()
+
+	return spec, nil
+}
+
+func WithSpecID(id int) PartitionOption {
+	return func(p *PartitionSpec) error {
+		p.id = id
+
+		return nil
+	}
+}
+
+func AddPartitionFieldBySourceID(sourceID int, targetName string, transform Transform, schema *Schema, fieldID *int) PartitionOption {
+	return func(p *PartitionSpec) error {
+		if schema == nil {
+			return errors.New("cannot add partition field with nil schema")
+		}
+		field, ok := schema.FindFieldByID(sourceID)
+		if !ok {
+			return fmt.Errorf("cannot find source column with id: %d in schema", sourceID)
+		}
+		err := addSpecFieldInternal(p, targetName, field, transform, fieldID)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	}
+}
+
+func addSpecFieldInternal(p *PartitionSpec, targetName string, field NestedField, transform Transform, fieldID *int) error {
+	if targetName == "" {
+		return errors.New("cannot use empty partition name")
+	}
+	for _, existingField := range p.fields {
+		if existingField.Name == targetName {
+			return errors.New("duplicate partition name: " + targetName)
+		}
+	}
+	var fieldIDValue int
+	if fieldID == nil {
+		fieldIDValue = unassignedFieldID
+	} else {
+		fieldIDValue = *fieldID
+	}
+	if err := p.checkForRedundantPartitions(field.ID, transform); err != nil {
+		return err
+	}
+	unboundField := PartitionField{
+		SourceID:  field.ID,
+		FieldID:   fieldIDValue,
+		Name:      targetName,
+		Transform: transform,
+	}
+	p.fields = append(p.fields, unboundField)
+
+	return nil
+}
+
+func (p *PartitionSpec) checkForRedundantPartitions(sourceID int, transform Transform) error {
+	for _, field := range p.fields {
+		if field.SourceID == sourceID && field.Transform.String() == transform.String() {
+			return fmt.Errorf("cannot add redundant partition with source id %d and transform %s. A partition with the same source id and transform already exists with name: %s", sourceID, transform, field.Name)
+		}
+	}
+
+	return nil
+}
+
+func (p *PartitionSpec) SetID(id int) {
+	p.id = id
+}
+
+func (ps *PartitionSpec) AssignPartitionFieldIds(lastAssignedFieldIDPtr *int) error {

Review Comment:
   Made it an unexported function + a `PartitionOption`, maybe don't even need that, we're maybe ok with just calling it from `PartitionSpec.BindToSchema`?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2334564140


##########
table/updates.go:
##########
@@ -553,20 +553,22 @@ func (u *removeSnapshotRefUpdate) Apply(builder *MetadataBuilder) error {
 
 type removeSpecUpdate struct {
 	baseUpdate
-	SpecIds []int64 `json:"spec-ids"`
+	SpecIds []int `json:"spec-ids"`

Review Comment:
   Then you should use `int32`. On a 64-bit system, `int` in Go is 64-bit. 😄 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "twuebi (via GitHub)" <gi...@apache.org>.
twuebi commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2337009926


##########
table/metadata.go:
##########
@@ -1278,63 +1384,117 @@ func NewMetadata(sc *iceberg.Schema, partitions *iceberg.PartitionSpec, sortOrde
 
 // NewMetadataWithUUID is like NewMetadata, but allows the caller to specify the UUID of the table rather than creating a new one.
 func NewMetadataWithUUID(sc *iceberg.Schema, partitions *iceberg.PartitionSpec, sortOrder SortOrder, location string, props iceberg.Properties, tableUuid uuid.UUID) (Metadata, error) {
-	freshSchema, err := iceberg.AssignFreshSchemaIDs(sc, nil)
+	if tableUuid == uuid.Nil {
+		tableUuid = uuid.New()
+	}
+	var err error
+	formatVersion := DefaultFormatVersion
+	if props != nil {
+		verStr, ok := props["format-version"]
+		if ok {
+			if formatVersion, err = strconv.Atoi(verStr); err != nil {
+				formatVersion = DefaultFormatVersion
+			}
+			delete(props, "format-version")
+		}
+	}
+
+	reassignedIds, err := reassignIDs(sc, partitions, sortOrder)
 	if err != nil {
 		return nil, err
 	}
 
-	freshPartitions, err := iceberg.AssignFreshPartitionSpecIDs(partitions, sc, freshSchema)
+	builder, err := NewMetadataBuilder()
 	if err != nil {
 		return nil, err
 	}
 
-	freshSortOrder, err := AssignFreshSortOrderIDs(sortOrder, sc, freshSchema)
+	_, err = builder.SetFormatVersion(formatVersion)
 	if err != nil {
 		return nil, err
 	}
 
-	if tableUuid == uuid.Nil {
-		tableUuid = uuid.New()
+	_, err = builder.SetUUID(tableUuid)
+	if err != nil {
+		return nil, err
 	}
 
-	formatVersion := DefaultFormatVersion
-	if props != nil {
-		verStr, ok := props["format-version"]
-		if ok {
-			if formatVersion, err = strconv.Atoi(verStr); err != nil {
-				formatVersion = DefaultFormatVersion
-			}
-			delete(props, "format-version")
+	_, err = builder.AddSortOrder(&reassignedIds.sortOrder, true)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = builder.SetDefaultSortOrderID(-1)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = builder.AddSchema(reassignedIds.schema)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = builder.SetCurrentSchemaID(-1)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = builder.AddPartitionSpec(reassignedIds.partitionSpec, true)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = builder.SetLoc(location)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = builder.SetProperties(props)
+	if err != nil {
+		return nil, err
+	}

Review Comment:
   We should make this a funct-opts thing eventually, I think. I went for the builder here so that there's a single place of validation



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "twuebi (via GitHub)" <gi...@apache.org>.
twuebi commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2334554729


##########
partitions.go:
##########
@@ -85,6 +87,130 @@ type PartitionSpec struct {
 	sourceIdToFields map[int][]PartitionField
 }
 
+type PartitionOption func(*PartitionSpec) error
+
+func NewPartitionSpecOpts(opts ...PartitionOption) (PartitionSpec, error) {
+	spec := PartitionSpec{
+		id: 0,
+	}
+	for _, opt := range opts {
+		if err := opt(&spec); err != nil {
+			return PartitionSpec{}, err
+		}
+	}
+	spec.initialize()
+
+	return spec, nil
+}
+
+func WithSpecID(id int) PartitionOption {
+	return func(p *PartitionSpec) error {
+		p.id = id
+
+		return nil
+	}
+}
+
+func AddPartitionFieldBySourceID(sourceID int, targetName string, transform Transform, schema *Schema, fieldID *int) PartitionOption {
+	return func(p *PartitionSpec) error {
+		if schema == nil {
+			return errors.New("cannot add partition field with nil schema")
+		}
+		field, ok := schema.FindFieldByID(sourceID)
+		if !ok {
+			return fmt.Errorf("cannot find source column with id: %d in schema", sourceID)
+		}
+		err := addSpecFieldInternal(p, targetName, field, transform, fieldID)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	}
+}
+
+func addSpecFieldInternal(p *PartitionSpec, targetName string, field NestedField, transform Transform, fieldID *int) error {

Review Comment:
   done



##########
partitions.go:
##########
@@ -85,6 +87,130 @@ type PartitionSpec struct {
 	sourceIdToFields map[int][]PartitionField
 }
 
+type PartitionOption func(*PartitionSpec) error
+
+func NewPartitionSpecOpts(opts ...PartitionOption) (PartitionSpec, error) {
+	spec := PartitionSpec{
+		id: 0,
+	}
+	for _, opt := range opts {
+		if err := opt(&spec); err != nil {
+			return PartitionSpec{}, err
+		}
+	}
+	spec.initialize()
+
+	return spec, nil
+}
+
+func WithSpecID(id int) PartitionOption {
+	return func(p *PartitionSpec) error {
+		p.id = id
+
+		return nil
+	}
+}
+
+func AddPartitionFieldBySourceID(sourceID int, targetName string, transform Transform, schema *Schema, fieldID *int) PartitionOption {
+	return func(p *PartitionSpec) error {
+		if schema == nil {
+			return errors.New("cannot add partition field with nil schema")
+		}
+		field, ok := schema.FindFieldByID(sourceID)
+		if !ok {
+			return fmt.Errorf("cannot find source column with id: %d in schema", sourceID)
+		}
+		err := addSpecFieldInternal(p, targetName, field, transform, fieldID)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	}
+}
+
+func addSpecFieldInternal(p *PartitionSpec, targetName string, field NestedField, transform Transform, fieldID *int) error {
+	if targetName == "" {
+		return errors.New("cannot use empty partition name")
+	}
+	for _, existingField := range p.fields {
+		if existingField.Name == targetName {
+			return errors.New("duplicate partition name: " + targetName)
+		}
+	}
+	var fieldIDValue int
+	if fieldID == nil {
+		fieldIDValue = unassignedFieldID
+	} else {
+		fieldIDValue = *fieldID
+	}
+	if err := p.checkForRedundantPartitions(field.ID, transform); err != nil {
+		return err
+	}
+	unboundField := PartitionField{
+		SourceID:  field.ID,
+		FieldID:   fieldIDValue,
+		Name:      targetName,
+		Transform: transform,
+	}
+	p.fields = append(p.fields, unboundField)
+
+	return nil
+}
+
+func (p *PartitionSpec) checkForRedundantPartitions(sourceID int, transform Transform) error {
+	for _, field := range p.fields {
+		if field.SourceID == sourceID && field.Transform.String() == transform.String() {
+			return fmt.Errorf("cannot add redundant partition with source id %d and transform %s. A partition with the same source id and transform already exists with name: %s", sourceID, transform, field.Name)

Review Comment:
   done



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "twuebi (via GitHub)" <gi...@apache.org>.
twuebi commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2334550099


##########
table/metadata.go:
##########
@@ -1234,6 +1359,17 @@ type metadataV2 struct {
 	commonMetadata
 }
 
+func initMetadataV2Deser() *metadataV2 {
+	return &metadataV2{
+		LastSeqNum:     -1,
+		commonMetadata: initCommonMetadataForDeserialization(),
+	}
+}
+
+func (m *metadataV2) FormatVersion() int {
+	return 2
+}

Review Comment:
   removed



##########
table/metadata.go:
##########
@@ -1156,6 +1255,18 @@ type metadataV1 struct {
 	commonMetadata
 }
 
+func initMetadataV1Deser() *metadataV1 {
+	return &metadataV1{
+		Schema:         nil,
+		Partition:      nil,

Review Comment:
   removed



##########
table/metadata.go:
##########
@@ -890,6 +960,29 @@ type commonMetadata struct {
 	SnapshotRefs       map[string]SnapshotRef  `json:"refs,omitempty"`
 }
 
+func initCommonMetadataForDeserialization() commonMetadata {
+	return commonMetadata{
+		FormatVersion:      0,
+		UUID:               uuid.UUID{},
+		Loc:                "",
+		LastUpdatedMS:      -1,
+		LastColumnId:       -1,
+		SchemaList:         nil,
+		CurrentSchemaID:    -1,
+		Specs:              nil,
+		DefaultSpecID:      -1,
+		LastPartitionID:    nil,
+		Props:              nil,
+		SnapshotList:       nil,
+		CurrentSnapshotID:  nil,
+		SnapshotLog:        nil,
+		MetadataLog:        nil,
+		SortOrderList:      nil,
+		DefaultSortOrderID: 0,
+		SnapshotRefs:       nil,
+	}

Review Comment:
   removed



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "twuebi (via GitHub)" <gi...@apache.org>.
twuebi commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2334015752


##########
partitions.go:
##########
@@ -85,6 +87,130 @@ type PartitionSpec struct {
 	sourceIdToFields map[int][]PartitionField
 }
 
+type PartitionOption func(*PartitionSpec) error
+
+func NewPartitionSpecOpts(opts ...PartitionOption) (PartitionSpec, error) {
+	spec := PartitionSpec{
+		id: 0,
+	}
+	for _, opt := range opts {
+		if err := opt(&spec); err != nil {
+			return PartitionSpec{}, err
+		}
+	}
+	spec.initialize()
+
+	return spec, nil
+}
+
+func WithSpecID(id int) PartitionOption {
+	return func(p *PartitionSpec) error {
+		p.id = id
+
+		return nil
+	}
+}
+
+func AddPartitionFieldBySourceID(sourceID int, targetName string, transform Transform, schema *Schema, fieldID *int) PartitionOption {
+	return func(p *PartitionSpec) error {
+		if schema == nil {
+			return errors.New("cannot add partition field with nil schema")
+		}
+		field, ok := schema.FindFieldByID(sourceID)
+		if !ok {
+			return fmt.Errorf("cannot find source column with id: %d in schema", sourceID)
+		}
+		err := addSpecFieldInternal(p, targetName, field, transform, fieldID)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	}
+}
+
+func addSpecFieldInternal(p *PartitionSpec, targetName string, field NestedField, transform Transform, fieldID *int) error {
+	if targetName == "" {
+		return errors.New("cannot use empty partition name")
+	}
+	for _, existingField := range p.fields {
+		if existingField.Name == targetName {
+			return errors.New("duplicate partition name: " + targetName)
+		}
+	}
+	var fieldIDValue int
+	if fieldID == nil {
+		fieldIDValue = unassignedFieldID
+	} else {
+		fieldIDValue = *fieldID
+	}
+	if err := p.checkForRedundantPartitions(field.ID, transform); err != nil {
+		return err
+	}
+	unboundField := PartitionField{
+		SourceID:  field.ID,
+		FieldID:   fieldIDValue,
+		Name:      targetName,
+		Transform: transform,
+	}
+	p.fields = append(p.fields, unboundField)
+
+	return nil
+}
+
+func (p *PartitionSpec) checkForRedundantPartitions(sourceID int, transform Transform) error {
+	for _, field := range p.fields {
+		if field.SourceID == sourceID && field.Transform.String() == transform.String() {
+			return fmt.Errorf("cannot add redundant partition with source id %d and transform %s. A partition with the same source id and transform already exists with name: %s", sourceID, transform, field.Name)
+		}
+	}
+
+	return nil
+}
+
+func (p *PartitionSpec) SetID(id int) {
+	p.id = id
+}
+
+func (ps *PartitionSpec) AssignPartitionFieldIds(lastAssignedFieldIDPtr *int) error {
+	// This is set_field_ids from iceberg-rust
+	// Already assigned partition ids. If we see one of these during iteration,
+	// we skip it.
+	assignedIds := make(map[int]struct{})
+	for _, field := range ps.fields {
+		if field.FieldID != unassignedFieldID {
+			if _, exists := assignedIds[field.FieldID]; exists {
+				return fmt.Errorf("duplicate field ID provided: %d", field.FieldID)
+			}
+			assignedIds[field.FieldID] = struct{}{}
+		}
+	}

Review Comment:
   Ah, forgot the actual reason: we don't have access to `lastPartitionID` in either of those places



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2356510232


##########
catalog/rest/rest.go:
##########
@@ -750,27 +750,12 @@ func (r *Catalog) CreateTable(ctx context.Context, identifier table.Identifier,
 		o(&cfg)
 	}
 
-	freshSchema, err := iceberg.AssignFreshSchemaIDs(schema, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	freshPartitionSpec, err := iceberg.AssignFreshPartitionSpecIDs(cfg.PartitionSpec, schema, freshSchema)
-	if err != nil {
-		return nil, err
-	}
-
-	freshSortOrder, err := table.AssignFreshSortOrderIDs(cfg.SortOrder, schema, freshSchema)
-	if err != nil {
-		return nil, err
-	}

Review Comment:
   I'll rerun the test on main, but it doesn't appear to be failing. The test failing with these changes is likely due to the field/partition/schema ID values



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2356581911


##########
catalog/rest/rest.go:
##########
@@ -750,27 +750,12 @@ func (r *Catalog) CreateTable(ctx context.Context, identifier table.Identifier,
 		o(&cfg)
 	}
 
-	freshSchema, err := iceberg.AssignFreshSchemaIDs(schema, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	freshPartitionSpec, err := iceberg.AssignFreshPartitionSpecIDs(cfg.PartitionSpec, schema, freshSchema)
-	if err != nil {
-		return nil, err
-	}
-
-	freshSortOrder, err := table.AssignFreshSortOrderIDs(cfg.SortOrder, schema, freshSchema)
-	if err != nil {
-		return nil, err
-	}

Review Comment:
   eventually we can shift to jsonv2 which will use an empty slice as the default instead of nil



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "twuebi (via GitHub)" <gi...@apache.org>.
twuebi commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2333992081


##########
partitions.go:
##########
@@ -85,6 +87,130 @@ type PartitionSpec struct {
 	sourceIdToFields map[int][]PartitionField
 }
 
+type PartitionOption func(*PartitionSpec) error
+
+func NewPartitionSpecOpts(opts ...PartitionOption) (PartitionSpec, error) {
+	spec := PartitionSpec{
+		id: 0,
+	}
+	for _, opt := range opts {
+		if err := opt(&spec); err != nil {
+			return PartitionSpec{}, err
+		}
+	}
+	spec.initialize()
+
+	return spec, nil
+}
+
+func WithSpecID(id int) PartitionOption {
+	return func(p *PartitionSpec) error {
+		p.id = id
+
+		return nil
+	}
+}
+
+func AddPartitionFieldBySourceID(sourceID int, targetName string, transform Transform, schema *Schema, fieldID *int) PartitionOption {
+	return func(p *PartitionSpec) error {
+		if schema == nil {
+			return errors.New("cannot add partition field with nil schema")
+		}
+		field, ok := schema.FindFieldByID(sourceID)
+		if !ok {
+			return fmt.Errorf("cannot find source column with id: %d in schema", sourceID)
+		}
+		err := addSpecFieldInternal(p, targetName, field, transform, fieldID)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	}
+}
+
+func addSpecFieldInternal(p *PartitionSpec, targetName string, field NestedField, transform Transform, fieldID *int) error {
+	if targetName == "" {
+		return errors.New("cannot use empty partition name")
+	}
+	for _, existingField := range p.fields {
+		if existingField.Name == targetName {
+			return errors.New("duplicate partition name: " + targetName)
+		}
+	}
+	var fieldIDValue int
+	if fieldID == nil {
+		fieldIDValue = unassignedFieldID
+	} else {
+		fieldIDValue = *fieldID
+	}
+	if err := p.checkForRedundantPartitions(field.ID, transform); err != nil {
+		return err
+	}
+	unboundField := PartitionField{
+		SourceID:  field.ID,
+		FieldID:   fieldIDValue,
+		Name:      targetName,
+		Transform: transform,
+	}
+	p.fields = append(p.fields, unboundField)
+
+	return nil
+}
+
+func (p *PartitionSpec) checkForRedundantPartitions(sourceID int, transform Transform) error {
+	for _, field := range p.fields {
+		if field.SourceID == sourceID && field.Transform.String() == transform.String() {
+			return fmt.Errorf("cannot add redundant partition with source id %d and transform %s. A partition with the same source id and transform already exists with name: %s", sourceID, transform, field.Name)
+		}
+	}
+
+	return nil
+}
+
+func (p *PartitionSpec) SetID(id int) {
+	p.id = id
+}
+
+func (ps *PartitionSpec) AssignPartitionFieldIds(lastAssignedFieldIDPtr *int) error {

Review Comment:
   Made it an unexpected function + a `PartitionOption`, maybe don't even need that, we're maybe ok with just calling it from `PartitionSpec.BindToSchema`?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2331159864


##########
partitions.go:
##########
@@ -85,6 +87,130 @@ type PartitionSpec struct {
 	sourceIdToFields map[int][]PartitionField
 }
 
+type PartitionOption func(*PartitionSpec) error
+
+func NewPartitionSpecOpts(opts ...PartitionOption) (PartitionSpec, error) {
+	spec := PartitionSpec{
+		id: 0,
+	}
+	for _, opt := range opts {
+		if err := opt(&spec); err != nil {
+			return PartitionSpec{}, err
+		}
+	}
+	spec.initialize()
+
+	return spec, nil
+}
+
+func WithSpecID(id int) PartitionOption {
+	return func(p *PartitionSpec) error {
+		p.id = id
+
+		return nil
+	}
+}
+
+func AddPartitionFieldBySourceID(sourceID int, targetName string, transform Transform, schema *Schema, fieldID *int) PartitionOption {
+	return func(p *PartitionSpec) error {
+		if schema == nil {
+			return errors.New("cannot add partition field with nil schema")
+		}
+		field, ok := schema.FindFieldByID(sourceID)
+		if !ok {
+			return fmt.Errorf("cannot find source column with id: %d in schema", sourceID)
+		}
+		err := addSpecFieldInternal(p, targetName, field, transform, fieldID)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	}
+}
+
+func addSpecFieldInternal(p *PartitionSpec, targetName string, field NestedField, transform Transform, fieldID *int) error {
+	if targetName == "" {
+		return errors.New("cannot use empty partition name")
+	}
+	for _, existingField := range p.fields {
+		if existingField.Name == targetName {
+			return errors.New("duplicate partition name: " + targetName)
+		}
+	}
+	var fieldIDValue int
+	if fieldID == nil {
+		fieldIDValue = unassignedFieldID
+	} else {
+		fieldIDValue = *fieldID
+	}
+	if err := p.checkForRedundantPartitions(field.ID, transform); err != nil {
+		return err
+	}
+	unboundField := PartitionField{
+		SourceID:  field.ID,
+		FieldID:   fieldIDValue,
+		Name:      targetName,
+		Transform: transform,
+	}
+	p.fields = append(p.fields, unboundField)
+
+	return nil
+}
+
+func (p *PartitionSpec) checkForRedundantPartitions(sourceID int, transform Transform) error {
+	for _, field := range p.fields {
+		if field.SourceID == sourceID && field.Transform.String() == transform.String() {
+			return fmt.Errorf("cannot add redundant partition with source id %d and transform %s. A partition with the same source id and transform already exists with name: %s", sourceID, transform, field.Name)
+		}
+	}
+
+	return nil
+}
+
+func (p *PartitionSpec) SetID(id int) {
+	p.id = id
+}

Review Comment:
   if we're adding a `SetID` function, we might as well just make the `ID` field a public field instead of having `SetID` and `ID()` methods. That's more idiomatic for Go.
   
   That said, I would rather this instead be a `WithID(id int) *PartitionSpec` method which returns a copy of the partition spec with the updated ID. That way we maintain that the PartitionSpec object itself can still be immutable



##########
partitions.go:
##########
@@ -85,6 +87,130 @@ type PartitionSpec struct {
 	sourceIdToFields map[int][]PartitionField
 }
 
+type PartitionOption func(*PartitionSpec) error
+
+func NewPartitionSpecOpts(opts ...PartitionOption) (PartitionSpec, error) {
+	spec := PartitionSpec{
+		id: 0,
+	}
+	for _, opt := range opts {
+		if err := opt(&spec); err != nil {
+			return PartitionSpec{}, err
+		}
+	}
+	spec.initialize()
+
+	return spec, nil
+}
+
+func WithSpecID(id int) PartitionOption {
+	return func(p *PartitionSpec) error {
+		p.id = id
+
+		return nil
+	}
+}
+
+func AddPartitionFieldBySourceID(sourceID int, targetName string, transform Transform, schema *Schema, fieldID *int) PartitionOption {
+	return func(p *PartitionSpec) error {
+		if schema == nil {
+			return errors.New("cannot add partition field with nil schema")
+		}
+		field, ok := schema.FindFieldByID(sourceID)
+		if !ok {
+			return fmt.Errorf("cannot find source column with id: %d in schema", sourceID)
+		}
+		err := addSpecFieldInternal(p, targetName, field, transform, fieldID)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	}
+}
+
+func addSpecFieldInternal(p *PartitionSpec, targetName string, field NestedField, transform Transform, fieldID *int) error {
+	if targetName == "" {
+		return errors.New("cannot use empty partition name")
+	}
+	for _, existingField := range p.fields {
+		if existingField.Name == targetName {
+			return errors.New("duplicate partition name: " + targetName)
+		}
+	}
+	var fieldIDValue int
+	if fieldID == nil {
+		fieldIDValue = unassignedFieldID
+	} else {
+		fieldIDValue = *fieldID
+	}
+	if err := p.checkForRedundantPartitions(field.ID, transform); err != nil {
+		return err
+	}
+	unboundField := PartitionField{
+		SourceID:  field.ID,
+		FieldID:   fieldIDValue,
+		Name:      targetName,
+		Transform: transform,
+	}
+	p.fields = append(p.fields, unboundField)
+
+	return nil
+}
+
+func (p *PartitionSpec) checkForRedundantPartitions(sourceID int, transform Transform) error {
+	for _, field := range p.fields {
+		if field.SourceID == sourceID && field.Transform.String() == transform.String() {
+			return fmt.Errorf("cannot add redundant partition with source id %d and transform %s. A partition with the same source id and transform already exists with name: %s", sourceID, transform, field.Name)
+		}
+	}
+
+	return nil
+}
+
+func (p *PartitionSpec) SetID(id int) {
+	p.id = id
+}
+
+func (ps *PartitionSpec) AssignPartitionFieldIds(lastAssignedFieldIDPtr *int) error {

Review Comment:
   same argument as above, I'd rather keep a PartitionSpec as an immutable object, so my preference is this should return a copy of the PartitionSpec with the new IDs



##########
table/metadata.go:
##########
@@ -1234,6 +1359,17 @@ type metadataV2 struct {
 	commonMetadata
 }
 
+func initMetadataV2Deser() *metadataV2 {
+	return &metadataV2{
+		LastSeqNum:     -1,
+		commonMetadata: initCommonMetadataForDeserialization(),
+	}
+}
+
+func (m *metadataV2) FormatVersion() int {
+	return 2
+}

Review Comment:
   common metadata already has a `Version` method which returns the `FormatVersion` value, why did we need a new method?



##########
table/metadata.go:
##########
@@ -890,6 +960,29 @@ type commonMetadata struct {
 	SnapshotRefs       map[string]SnapshotRef  `json:"refs,omitempty"`
 }
 
+func initCommonMetadataForDeserialization() commonMetadata {
+	return commonMetadata{
+		FormatVersion:      0,
+		UUID:               uuid.UUID{},
+		Loc:                "",
+		LastUpdatedMS:      -1,
+		LastColumnId:       -1,
+		SchemaList:         nil,
+		CurrentSchemaID:    -1,
+		Specs:              nil,
+		DefaultSpecID:      -1,
+		LastPartitionID:    nil,
+		Props:              nil,
+		SnapshotList:       nil,
+		CurrentSnapshotID:  nil,
+		SnapshotLog:        nil,
+		MetadataLog:        nil,
+		SortOrderList:      nil,
+		DefaultSortOrderID: 0,
+		SnapshotRefs:       nil,
+	}

Review Comment:
   simplify this by removing everything that is just being set to the zero value. i.e.: you only need to include the things that are being set to -1.



##########
partitions.go:
##########
@@ -85,6 +87,130 @@ type PartitionSpec struct {
 	sourceIdToFields map[int][]PartitionField
 }
 
+type PartitionOption func(*PartitionSpec) error
+
+func NewPartitionSpecOpts(opts ...PartitionOption) (PartitionSpec, error) {
+	spec := PartitionSpec{
+		id: 0,
+	}
+	for _, opt := range opts {
+		if err := opt(&spec); err != nil {
+			return PartitionSpec{}, err
+		}
+	}
+	spec.initialize()
+
+	return spec, nil
+}
+
+func WithSpecID(id int) PartitionOption {
+	return func(p *PartitionSpec) error {
+		p.id = id
+
+		return nil
+	}
+}
+
+func AddPartitionFieldBySourceID(sourceID int, targetName string, transform Transform, schema *Schema, fieldID *int) PartitionOption {
+	return func(p *PartitionSpec) error {
+		if schema == nil {
+			return errors.New("cannot add partition field with nil schema")
+		}
+		field, ok := schema.FindFieldByID(sourceID)
+		if !ok {
+			return fmt.Errorf("cannot find source column with id: %d in schema", sourceID)
+		}
+		err := addSpecFieldInternal(p, targetName, field, transform, fieldID)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	}
+}
+
+func addSpecFieldInternal(p *PartitionSpec, targetName string, field NestedField, transform Transform, fieldID *int) error {
+	if targetName == "" {
+		return errors.New("cannot use empty partition name")
+	}
+	for _, existingField := range p.fields {
+		if existingField.Name == targetName {
+			return errors.New("duplicate partition name: " + targetName)
+		}
+	}
+	var fieldIDValue int
+	if fieldID == nil {
+		fieldIDValue = unassignedFieldID
+	} else {
+		fieldIDValue = *fieldID
+	}
+	if err := p.checkForRedundantPartitions(field.ID, transform); err != nil {
+		return err
+	}
+	unboundField := PartitionField{
+		SourceID:  field.ID,
+		FieldID:   fieldIDValue,
+		Name:      targetName,
+		Transform: transform,
+	}
+	p.fields = append(p.fields, unboundField)
+
+	return nil
+}
+
+func (p *PartitionSpec) checkForRedundantPartitions(sourceID int, transform Transform) error {
+	for _, field := range p.fields {
+		if field.SourceID == sourceID && field.Transform.String() == transform.String() {
+			return fmt.Errorf("cannot add redundant partition with source id %d and transform %s. A partition with the same source id and transform already exists with name: %s", sourceID, transform, field.Name)
+		}
+	}
+
+	return nil
+}
+
+func (p *PartitionSpec) SetID(id int) {
+	p.id = id
+}
+
+func (ps *PartitionSpec) AssignPartitionFieldIds(lastAssignedFieldIDPtr *int) error {
+	// This is set_field_ids from iceberg-rust
+	// Already assigned partition ids. If we see one of these during iteration,
+	// we skip it.
+	assignedIds := make(map[int]struct{})
+	for _, field := range ps.fields {
+		if field.FieldID != unassignedFieldID {
+			if _, exists := assignedIds[field.FieldID]; exists {
+				return fmt.Errorf("duplicate field ID provided: %d", field.FieldID)
+			}
+			assignedIds[field.FieldID] = struct{}{}
+		}
+	}
+
+	lastAssignedFieldID := ps.LastAssignedFieldID()
+	if lastAssignedFieldIDPtr != nil {
+		lastAssignedFieldID = *lastAssignedFieldIDPtr
+	}

Review Comment:
   instead of taking a pointer, can we just take an `int` and if the value is `<= 0` we use the current `LastAssignedFieldID()`?



##########
partitions.go:
##########
@@ -85,6 +87,130 @@ type PartitionSpec struct {
 	sourceIdToFields map[int][]PartitionField
 }
 
+type PartitionOption func(*PartitionSpec) error
+
+func NewPartitionSpecOpts(opts ...PartitionOption) (PartitionSpec, error) {
+	spec := PartitionSpec{
+		id: 0,
+	}
+	for _, opt := range opts {
+		if err := opt(&spec); err != nil {
+			return PartitionSpec{}, err
+		}
+	}
+	spec.initialize()
+
+	return spec, nil
+}
+
+func WithSpecID(id int) PartitionOption {
+	return func(p *PartitionSpec) error {
+		p.id = id
+
+		return nil
+	}
+}
+
+func AddPartitionFieldBySourceID(sourceID int, targetName string, transform Transform, schema *Schema, fieldID *int) PartitionOption {
+	return func(p *PartitionSpec) error {
+		if schema == nil {
+			return errors.New("cannot add partition field with nil schema")
+		}
+		field, ok := schema.FindFieldByID(sourceID)
+		if !ok {
+			return fmt.Errorf("cannot find source column with id: %d in schema", sourceID)
+		}
+		err := addSpecFieldInternal(p, targetName, field, transform, fieldID)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	}
+}
+
+func addSpecFieldInternal(p *PartitionSpec, targetName string, field NestedField, transform Transform, fieldID *int) error {
+	if targetName == "" {
+		return errors.New("cannot use empty partition name")
+	}
+	for _, existingField := range p.fields {
+		if existingField.Name == targetName {
+			return errors.New("duplicate partition name: " + targetName)
+		}
+	}
+	var fieldIDValue int
+	if fieldID == nil {
+		fieldIDValue = unassignedFieldID
+	} else {
+		fieldIDValue = *fieldID
+	}
+	if err := p.checkForRedundantPartitions(field.ID, transform); err != nil {
+		return err
+	}
+	unboundField := PartitionField{
+		SourceID:  field.ID,
+		FieldID:   fieldIDValue,
+		Name:      targetName,
+		Transform: transform,
+	}
+	p.fields = append(p.fields, unboundField)
+
+	return nil
+}
+
+func (p *PartitionSpec) checkForRedundantPartitions(sourceID int, transform Transform) error {
+	for _, field := range p.fields {
+		if field.SourceID == sourceID && field.Transform.String() == transform.String() {
+			return fmt.Errorf("cannot add redundant partition with source id %d and transform %s. A partition with the same source id and transform already exists with name: %s", sourceID, transform, field.Name)

Review Comment:
   put the remaining arguments on a new line to reduce the line wrap and length. Also, we already have a mapping of source-id to the field, so you can do 
   
   ```go
   if fields, ok := p.sourceIdToFields[sourceID]; ok {
      for _, f := range fields {
           if f.Transform.Equals(transform) {
               return fmt.Errorf(.....)
           }
       }
   }
   
   return nil
   ```
   
   instead
   
     



##########
partitions.go:
##########
@@ -85,6 +87,130 @@ type PartitionSpec struct {
 	sourceIdToFields map[int][]PartitionField
 }
 
+type PartitionOption func(*PartitionSpec) error
+
+func NewPartitionSpecOpts(opts ...PartitionOption) (PartitionSpec, error) {
+	spec := PartitionSpec{
+		id: 0,
+	}
+	for _, opt := range opts {
+		if err := opt(&spec); err != nil {
+			return PartitionSpec{}, err
+		}
+	}
+	spec.initialize()
+
+	return spec, nil
+}
+
+func WithSpecID(id int) PartitionOption {
+	return func(p *PartitionSpec) error {
+		p.id = id
+
+		return nil
+	}
+}
+
+func AddPartitionFieldBySourceID(sourceID int, targetName string, transform Transform, schema *Schema, fieldID *int) PartitionOption {
+	return func(p *PartitionSpec) error {
+		if schema == nil {
+			return errors.New("cannot add partition field with nil schema")
+		}
+		field, ok := schema.FindFieldByID(sourceID)
+		if !ok {
+			return fmt.Errorf("cannot find source column with id: %d in schema", sourceID)
+		}
+		err := addSpecFieldInternal(p, targetName, field, transform, fieldID)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	}
+}
+
+func addSpecFieldInternal(p *PartitionSpec, targetName string, field NestedField, transform Transform, fieldID *int) error {

Review Comment:
   why not make this a method on `PartitionSpec`?



##########
table/metadata.go:
##########
@@ -1156,6 +1255,18 @@ type metadataV1 struct {
 	commonMetadata
 }
 
+func initMetadataV1Deser() *metadataV1 {
+	return &metadataV1{
+		Schema:         nil,
+		Partition:      nil,

Review Comment:
   same comment as above, don't include these if you're just setting them to the zero value



##########
partitions.go:
##########
@@ -85,6 +87,130 @@ type PartitionSpec struct {
 	sourceIdToFields map[int][]PartitionField
 }
 
+type PartitionOption func(*PartitionSpec) error
+
+func NewPartitionSpecOpts(opts ...PartitionOption) (PartitionSpec, error) {

Review Comment:
   these need docstring comments to explain why someone would use this instead of `NewPartitionSpec` etc.



##########
partitions.go:
##########
@@ -85,6 +87,130 @@ type PartitionSpec struct {
 	sourceIdToFields map[int][]PartitionField
 }
 
+type PartitionOption func(*PartitionSpec) error
+
+func NewPartitionSpecOpts(opts ...PartitionOption) (PartitionSpec, error) {
+	spec := PartitionSpec{
+		id: 0,
+	}
+	for _, opt := range opts {
+		if err := opt(&spec); err != nil {
+			return PartitionSpec{}, err
+		}
+	}
+	spec.initialize()
+
+	return spec, nil
+}
+
+func WithSpecID(id int) PartitionOption {
+	return func(p *PartitionSpec) error {
+		p.id = id
+
+		return nil
+	}
+}
+
+func AddPartitionFieldBySourceID(sourceID int, targetName string, transform Transform, schema *Schema, fieldID *int) PartitionOption {
+	return func(p *PartitionSpec) error {
+		if schema == nil {
+			return errors.New("cannot add partition field with nil schema")
+		}
+		field, ok := schema.FindFieldByID(sourceID)
+		if !ok {
+			return fmt.Errorf("cannot find source column with id: %d in schema", sourceID)
+		}
+		err := addSpecFieldInternal(p, targetName, field, transform, fieldID)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	}
+}
+
+func addSpecFieldInternal(p *PartitionSpec, targetName string, field NestedField, transform Transform, fieldID *int) error {
+	if targetName == "" {
+		return errors.New("cannot use empty partition name")
+	}
+	for _, existingField := range p.fields {
+		if existingField.Name == targetName {
+			return errors.New("duplicate partition name: " + targetName)
+		}
+	}
+	var fieldIDValue int
+	if fieldID == nil {
+		fieldIDValue = unassignedFieldID
+	} else {
+		fieldIDValue = *fieldID
+	}
+	if err := p.checkForRedundantPartitions(field.ID, transform); err != nil {
+		return err
+	}
+	unboundField := PartitionField{
+		SourceID:  field.ID,
+		FieldID:   fieldIDValue,
+		Name:      targetName,
+		Transform: transform,
+	}
+	p.fields = append(p.fields, unboundField)
+
+	return nil
+}
+
+func (p *PartitionSpec) checkForRedundantPartitions(sourceID int, transform Transform) error {
+	for _, field := range p.fields {
+		if field.SourceID == sourceID && field.Transform.String() == transform.String() {
+			return fmt.Errorf("cannot add redundant partition with source id %d and transform %s. A partition with the same source id and transform already exists with name: %s", sourceID, transform, field.Name)
+		}
+	}
+
+	return nil
+}
+
+func (p *PartitionSpec) SetID(id int) {
+	p.id = id
+}
+
+func (ps *PartitionSpec) AssignPartitionFieldIds(lastAssignedFieldIDPtr *int) error {
+	// This is set_field_ids from iceberg-rust
+	// Already assigned partition ids. If we see one of these during iteration,
+	// we skip it.
+	assignedIds := make(map[int]struct{})
+	for _, field := range ps.fields {
+		if field.FieldID != unassignedFieldID {
+			if _, exists := assignedIds[field.FieldID]; exists {
+				return fmt.Errorf("duplicate field ID provided: %d", field.FieldID)
+			}
+			assignedIds[field.FieldID] = struct{}{}
+		}
+	}

Review Comment:
   shouldn't this get done during initialization or during `NewPartitionSpec` instead of here?



##########
table/metadata.go:
##########
@@ -134,6 +134,8 @@ type Metadata interface {
 	NameMapping() iceberg.NameMapping
 
 	LastSequenceNumber() int64
+
+	FormatVersion() int

Review Comment:
   there's already `Version()` why do we need this also?



##########
table/updates.go:
##########
@@ -553,20 +553,22 @@ func (u *removeSnapshotRefUpdate) Apply(builder *MetadataBuilder) error {
 
 type removeSpecUpdate struct {
 	baseUpdate
-	SpecIds []int64 `json:"spec-ids"`
+	SpecIds []int `json:"spec-ids"`

Review Comment:
   why this change?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "twuebi (via GitHub)" <gi...@apache.org>.
twuebi commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2331209855


##########
partitions.go:
##########
@@ -30,6 +31,7 @@ import (
 const (
 	PartitionDataIDStart   = 1000
 	InitialPartitionSpecID = 0
+	unassignedFieldID      = 0

Review Comment:
   field-ids semantic space starts at 1000, so 0 for unassigned is fine? Can also change to -1 if preferred



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "twuebi (via GitHub)" <gi...@apache.org>.
twuebi commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2334553431


##########
partitions.go:
##########
@@ -85,6 +87,130 @@ type PartitionSpec struct {
 	sourceIdToFields map[int][]PartitionField
 }
 
+type PartitionOption func(*PartitionSpec) error
+
+func NewPartitionSpecOpts(opts ...PartitionOption) (PartitionSpec, error) {
+	spec := PartitionSpec{
+		id: 0,
+	}
+	for _, opt := range opts {
+		if err := opt(&spec); err != nil {
+			return PartitionSpec{}, err
+		}
+	}
+	spec.initialize()
+
+	return spec, nil
+}
+
+func WithSpecID(id int) PartitionOption {
+	return func(p *PartitionSpec) error {
+		p.id = id
+
+		return nil
+	}
+}
+
+func AddPartitionFieldBySourceID(sourceID int, targetName string, transform Transform, schema *Schema, fieldID *int) PartitionOption {
+	return func(p *PartitionSpec) error {
+		if schema == nil {
+			return errors.New("cannot add partition field with nil schema")
+		}
+		field, ok := schema.FindFieldByID(sourceID)
+		if !ok {
+			return fmt.Errorf("cannot find source column with id: %d in schema", sourceID)
+		}
+		err := addSpecFieldInternal(p, targetName, field, transform, fieldID)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	}
+}
+
+func addSpecFieldInternal(p *PartitionSpec, targetName string, field NestedField, transform Transform, fieldID *int) error {
+	if targetName == "" {
+		return errors.New("cannot use empty partition name")
+	}
+	for _, existingField := range p.fields {
+		if existingField.Name == targetName {
+			return errors.New("duplicate partition name: " + targetName)
+		}
+	}
+	var fieldIDValue int
+	if fieldID == nil {
+		fieldIDValue = unassignedFieldID
+	} else {
+		fieldIDValue = *fieldID
+	}
+	if err := p.checkForRedundantPartitions(field.ID, transform); err != nil {
+		return err
+	}
+	unboundField := PartitionField{
+		SourceID:  field.ID,
+		FieldID:   fieldIDValue,
+		Name:      targetName,
+		Transform: transform,
+	}
+	p.fields = append(p.fields, unboundField)
+
+	return nil
+}
+
+func (p *PartitionSpec) checkForRedundantPartitions(sourceID int, transform Transform) error {
+	for _, field := range p.fields {
+		if field.SourceID == sourceID && field.Transform.String() == transform.String() {
+			return fmt.Errorf("cannot add redundant partition with source id %d and transform %s. A partition with the same source id and transform already exists with name: %s", sourceID, transform, field.Name)
+		}
+	}
+
+	return nil
+}
+
+func (p *PartitionSpec) SetID(id int) {
+	p.id = id
+}
+
+func (ps *PartitionSpec) AssignPartitionFieldIds(lastAssignedFieldIDPtr *int) error {
+	// This is set_field_ids from iceberg-rust
+	// Already assigned partition ids. If we see one of these during iteration,
+	// we skip it.
+	assignedIds := make(map[int]struct{})
+	for _, field := range ps.fields {
+		if field.FieldID != unassignedFieldID {
+			if _, exists := assignedIds[field.FieldID]; exists {
+				return fmt.Errorf("duplicate field ID provided: %d", field.FieldID)
+			}
+			assignedIds[field.FieldID] = struct{}{}
+		}
+	}
+
+	lastAssignedFieldID := ps.LastAssignedFieldID()
+	if lastAssignedFieldIDPtr != nil {
+		lastAssignedFieldID = *lastAssignedFieldIDPtr
+	}

Review Comment:
   in other words, I miss [Option](https://doc.rust-lang.org/std/option/enum.Option.html) 😄 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "twuebi (via GitHub)" <gi...@apache.org>.
twuebi commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2337009926


##########
table/metadata.go:
##########
@@ -1278,63 +1384,117 @@ func NewMetadata(sc *iceberg.Schema, partitions *iceberg.PartitionSpec, sortOrde
 
 // NewMetadataWithUUID is like NewMetadata, but allows the caller to specify the UUID of the table rather than creating a new one.
 func NewMetadataWithUUID(sc *iceberg.Schema, partitions *iceberg.PartitionSpec, sortOrder SortOrder, location string, props iceberg.Properties, tableUuid uuid.UUID) (Metadata, error) {
-	freshSchema, err := iceberg.AssignFreshSchemaIDs(sc, nil)
+	if tableUuid == uuid.Nil {
+		tableUuid = uuid.New()
+	}
+	var err error
+	formatVersion := DefaultFormatVersion
+	if props != nil {
+		verStr, ok := props["format-version"]
+		if ok {
+			if formatVersion, err = strconv.Atoi(verStr); err != nil {
+				formatVersion = DefaultFormatVersion
+			}
+			delete(props, "format-version")
+		}
+	}
+
+	reassignedIds, err := reassignIDs(sc, partitions, sortOrder)
 	if err != nil {
 		return nil, err
 	}
 
-	freshPartitions, err := iceberg.AssignFreshPartitionSpecIDs(partitions, sc, freshSchema)
+	builder, err := NewMetadataBuilder()
 	if err != nil {
 		return nil, err
 	}
 
-	freshSortOrder, err := AssignFreshSortOrderIDs(sortOrder, sc, freshSchema)
+	_, err = builder.SetFormatVersion(formatVersion)
 	if err != nil {
 		return nil, err
 	}
 
-	if tableUuid == uuid.Nil {
-		tableUuid = uuid.New()
+	_, err = builder.SetUUID(tableUuid)
+	if err != nil {
+		return nil, err
 	}
 
-	formatVersion := DefaultFormatVersion
-	if props != nil {
-		verStr, ok := props["format-version"]
-		if ok {
-			if formatVersion, err = strconv.Atoi(verStr); err != nil {
-				formatVersion = DefaultFormatVersion
-			}
-			delete(props, "format-version")
+	_, err = builder.AddSortOrder(&reassignedIds.sortOrder, true)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = builder.SetDefaultSortOrderID(-1)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = builder.AddSchema(reassignedIds.schema)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = builder.SetCurrentSchemaID(-1)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = builder.AddPartitionSpec(reassignedIds.partitionSpec, true)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = builder.SetLoc(location)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = builder.SetProperties(props)
+	if err != nil {
+		return nil, err
+	}

Review Comment:
   We should make this a funct-opts thing eventually, I think



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "twuebi (via GitHub)" <gi...@apache.org>.
twuebi commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2334601152


##########
table/updates.go:
##########
@@ -553,20 +553,22 @@ func (u *removeSnapshotRefUpdate) Apply(builder *MetadataBuilder) error {
 
 type removeSpecUpdate struct {
 	baseUpdate
-	SpecIds []int64 `json:"spec-ids"`
+	SpecIds []int `json:"spec-ids"`

Review Comment:
   ughhh 😅 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "twuebi (via GitHub)" <gi...@apache.org>.
twuebi commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2337009926


##########
table/metadata.go:
##########
@@ -1278,63 +1384,117 @@ func NewMetadata(sc *iceberg.Schema, partitions *iceberg.PartitionSpec, sortOrde
 
 // NewMetadataWithUUID is like NewMetadata, but allows the caller to specify the UUID of the table rather than creating a new one.
 func NewMetadataWithUUID(sc *iceberg.Schema, partitions *iceberg.PartitionSpec, sortOrder SortOrder, location string, props iceberg.Properties, tableUuid uuid.UUID) (Metadata, error) {
-	freshSchema, err := iceberg.AssignFreshSchemaIDs(sc, nil)
+	if tableUuid == uuid.Nil {
+		tableUuid = uuid.New()
+	}
+	var err error
+	formatVersion := DefaultFormatVersion
+	if props != nil {
+		verStr, ok := props["format-version"]
+		if ok {
+			if formatVersion, err = strconv.Atoi(verStr); err != nil {
+				formatVersion = DefaultFormatVersion
+			}
+			delete(props, "format-version")
+		}
+	}
+
+	reassignedIds, err := reassignIDs(sc, partitions, sortOrder)
 	if err != nil {
 		return nil, err
 	}
 
-	freshPartitions, err := iceberg.AssignFreshPartitionSpecIDs(partitions, sc, freshSchema)
+	builder, err := NewMetadataBuilder()
 	if err != nil {
 		return nil, err
 	}
 
-	freshSortOrder, err := AssignFreshSortOrderIDs(sortOrder, sc, freshSchema)
+	_, err = builder.SetFormatVersion(formatVersion)
 	if err != nil {
 		return nil, err
 	}
 
-	if tableUuid == uuid.Nil {
-		tableUuid = uuid.New()
+	_, err = builder.SetUUID(tableUuid)
+	if err != nil {
+		return nil, err
 	}
 
-	formatVersion := DefaultFormatVersion
-	if props != nil {
-		verStr, ok := props["format-version"]
-		if ok {
-			if formatVersion, err = strconv.Atoi(verStr); err != nil {
-				formatVersion = DefaultFormatVersion
-			}
-			delete(props, "format-version")
+	_, err = builder.AddSortOrder(&reassignedIds.sortOrder, true)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = builder.SetDefaultSortOrderID(-1)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = builder.AddSchema(reassignedIds.schema)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = builder.SetCurrentSchemaID(-1)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = builder.AddPartitionSpec(reassignedIds.partitionSpec, true)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = builder.SetLoc(location)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = builder.SetProperties(props)
+	if err != nil {
+		return nil, err
+	}

Review Comment:
   We should make this a funct-opts thing eventually, I think. I went for the builder to replace the struct literal so that there's a single place of validation



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2356666438


##########
catalog/rest/rest.go:
##########
@@ -745,11 +745,15 @@ func (r *Catalog) CreateTable(ctx context.Context, identifier table.Identifier,
 		return nil, err
 	}
 
-	var cfg catalog.CreateTableCfg
+	cfg := catalog.NewCreateTableCfg()
 	for _, o := range opts {
 		o(&cfg)
 	}
 
+	if cfg.SortOrder.Fields == nil && cfg.SortOrder.OrderID == 0 {
+		cfg.SortOrder = table.UnsortedSortOrder
+	}

Review Comment:
   you shouldn't need this anymore if you have the `NewCreateTableCfg`



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] feat(metadata): PartitionSpec builder & validations [iceberg-go]

Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade commented on code in PR #558:
URL: https://github.com/apache/iceberg-go/pull/558#discussion_r2356090460


##########
catalog/rest/rest.go:
##########
@@ -750,27 +750,12 @@ func (r *Catalog) CreateTable(ctx context.Context, identifier table.Identifier,
 		o(&cfg)
 	}
 
-	freshSchema, err := iceberg.AssignFreshSchemaIDs(schema, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	freshPartitionSpec, err := iceberg.AssignFreshPartitionSpecIDs(cfg.PartitionSpec, schema, freshSchema)
-	if err != nil {
-		return nil, err
-	}
-
-	freshSortOrder, err := table.AssignFreshSortOrderIDs(cfg.SortOrder, schema, freshSchema)
-	if err != nil {
-		return nil, err
-	}

Review Comment:
   I think this is why the REST catalog integration is failing, maybe something needs to change (or get updated?) with the iceberg rest catalog image?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org