You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@iceberg.apache.org by "zeroshade (via GitHub)" <gi...@apache.org> on 2023/08/21 20:46:25 UTC
[GitHub] [iceberg-go] zeroshade opened a new pull request, #3: feat(manifests): Adding implementation of manifest files
zeroshade opened a new pull request, #3:
URL: https://github.com/apache/iceberg-go/pull/3
Adds an implementation for Manifest Lists, Manifest Entries, and Data Files along with interfaces for wrapping file system IO handling.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] nastra commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1333868102
##########
internal/avro_schemas.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package internal
+
+import "github.com/hamba/avro/v2"
+
+const (
+ ManifestListV1Key = "manifest-list-v1"
+ ManifestListV2Key = "manifest-list-v2"
+ ManifestEntryV1Key = "manifest-entry-v1"
+ ManifestEntryV2Key = "manifest-entry-v2"
+)
+
+var (
+ AvroSchemaCache avro.SchemaCache
+)
+
+func init() {
+ AvroSchemaCache.Add(ManifestListV1Key, avro.MustParse(`{
+ "type": "record",
+ "name": "manifest_file",
+ "fields": [
+ {"name": "manifest_path", "type": "string", "doc": "Location URI with FS scheme", "field-id": 500},
+ {"name": "manifest_length", "type": "long", "doc": "Total file size in bytes", "field-id": 501},
+ {"name": "partition_spec_id", "type": "int", "doc": "Spec ID used to write", "field-id": 502},
+ {
+ "name": "added_snapshot_id",
+ "type": "long",
Review Comment:
for V1 this one is optional
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] nastra commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1326999760
##########
manifest.go:
##########
@@ -0,0 +1,655 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package iceberg
+
+import (
+ "io"
+ "sync"
+
+ iceio "github.com/apache/iceberg-go/io"
+
+ "github.com/hamba/avro/v2/ocf"
+)
+
+// ManifestContent indicates the type of data inside of the files
+// described by a manifest. This will indicate whether the data files
+// contain active data or deleted rows.
+type ManifestContent int32
+
+const (
+ ManifestContentData ManifestContent = 0
+ ManifestContentDeletes ManifestContent = 1
+)
+
+type fieldSummary struct {
+ ContainsNull bool `avro:"contains_null"`
+ ContainsNaN *bool `avro:"contains_nan"`
+ LowerBound *[]byte `avro:"lower_bound"`
+ UpperBound *[]byte `avro:"upper_bound"`
+}
+
+type manifestFileV1 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ AddedSnapshotID *int64 `avro:"added_snapshot_id"`
+ AddedFilesCount *int32 `avro:"added_data_files_count"`
+ ExistingFilesCount *int32 `avro:"existing_data_files_count"`
+ DeletedFilesCount *int32 `avro:"deleted_data_files_count"`
+ AddedRowsCount *int64 `avro:"added_rows_count"`
+ ExistingRowsCount *int64 `avro:"existing_rows_count"`
+ DeletedRowsCount *int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV1) Version() int { return 1 }
+func (m *manifestFileV1) FilePath() string { return m.Path }
+func (m *manifestFileV1) Length() int64 { return m.Len }
+func (m *manifestFileV1) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV1) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV1) SnapshotID() int64 {
+ if m.AddedSnapshotID == nil {
+ return 0
+ }
+ return *m.AddedSnapshotID
+}
+
+func (m *manifestFileV1) AddedDataFiles() int32 {
+ if m.AddedFilesCount == nil {
+ return 0
+ }
+ return *m.AddedFilesCount
+}
+
+func (m *manifestFileV1) ExistingDataFiles() int32 {
+ if m.ExistingFilesCount == nil {
+ return 0
+ }
+ return *m.ExistingFilesCount
+}
+
+func (m *manifestFileV1) DeletedDataFiles() int32 {
+ if m.DeletedFilesCount == nil {
+ return 0
+ }
+ return *m.DeletedFilesCount
+}
+
+func (m *manifestFileV1) AddedRows() int64 {
+ if m.AddedRowsCount == nil {
+ return 0
+ }
+ return *m.AddedRowsCount
+}
+
+func (m *manifestFileV1) ExistingRows() int64 {
+ if m.ExistingRowsCount == nil {
+ return 0
+ }
+ return *m.ExistingRowsCount
+}
+
+func (m *manifestFileV1) DeletedRows() int64 {
+ if m.DeletedRowsCount == nil {
+ return 0
+ }
+ return *m.DeletedRowsCount
+}
+
+func (m *manifestFileV1) HasAddedFiles() bool {
+ return m.AddedFilesCount == nil || *m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV1) HasExistingFiles() bool {
+ return m.ExistingFilesCount == nil || *m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV1) SequenceNum() int64 { return 0 }
+func (m *manifestFileV1) MinSequenceNum() int64 { return 0 }
+func (m *manifestFileV1) Metadata() []byte { return m.KeyMetadata }
+func (m *manifestFileV1) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV1) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+type manifestFileV2 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ SeqNumber int64 `avro:"sequence_number"`
+ MinSeqNumber int64 `avro:"min_sequence_number"`
+ AddedSnapshotID int64 `avro:"added_snapshot_id"`
+ AddedFilesCount int32 `avro:"added_files_count"`
+ ExistingFilesCount int32 `avro:"existing_files_count"`
+ DeletedFilesCount int32 `avro:"deleted_files_count"`
+ AddedRowsCount int64 `avro:"added_rows_count"`
+ ExistingRowsCount int64 `avro:"existing_rows_count"`
+ DeletedRowsCount int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV2) Version() int { return 2 }
+
+func (m *manifestFileV2) FilePath() string { return m.Path }
+func (m *manifestFileV2) Length() int64 { return m.Len }
+func (m *manifestFileV2) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV2) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV2) SnapshotID() int64 {
+ return m.AddedSnapshotID
+}
+
+func (m *manifestFileV2) AddedDataFiles() int32 {
+ return m.AddedFilesCount
+}
+
+func (m *manifestFileV2) ExistingDataFiles() int32 {
+ return m.ExistingFilesCount
+}
+
+func (m *manifestFileV2) DeletedDataFiles() int32 {
+ return m.DeletedFilesCount
+}
+
+func (m *manifestFileV2) AddedRows() int64 {
+ return m.AddedRowsCount
+}
+
+func (m *manifestFileV2) ExistingRows() int64 {
+ return m.ExistingRowsCount
+}
+
+func (m *manifestFileV2) DeletedRows() int64 {
+ return m.DeletedRowsCount
+}
+
+func (m *manifestFileV2) SequenceNum() int64 { return m.SeqNumber }
+func (m *manifestFileV2) MinSequenceNum() int64 { return m.MinSeqNumber }
+func (m *manifestFileV2) Metadata() []byte { return m.KeyMetadata }
+
+func (m *manifestFileV2) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV2) HasAddedFiles() bool {
+ return m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV2) HasExistingFiles() bool {
+ return m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV2) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+func fetchManifestEntries(m ManifestFile, fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ f, err := fs.Open(m.FilePath())
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+
+ dec, err := ocf.NewDecoder(f)
+ if err != nil {
+ return nil, err
+ }
+
+ metadata := dec.Metadata()
+ isVer1 := true
+ if string(metadata["format-version"]) == "2" {
+ isVer1 = false
+ }
+
+ results := make([]ManifestEntry, 0)
+ for dec.HasNext() {
+ var tmp ManifestEntry
+ if isVer1 {
+ tmp = &manifestEntryV1{}
+ } else {
+ tmp = &manifestEntryV2{}
+ }
+
+ if err := dec.Decode(tmp); err != nil {
+ return nil, err
+ }
+
+ if !discardDeleted || tmp.Status() != EntryStatusDELETED {
+ tmp.inheritSeqNum(m)
+ results = append(results, tmp)
+ }
+ }
+
+ return results, dec.Error()
+}
+
+// ManifestFile is the interface which covers both V1 and V2 manifest files.
+type ManifestFile interface {
+ // Version returns the version number of this manifest file.
+ // It should be 1 or 2.
+ Version() int
+ // FilePath is the location URI of this manifest file.
+ FilePath() string
+ // Length is the length in bytes of the manifest file.
+ Length() int64
+ // PartitionID is the ID of the partition spec used to write
+ // this manifest. It must be listed in the table metadata
+ // partition-specs.
+ PartitionID() int32
+ // ManifestContent is the type of files tracked by this manifest,
+ // either data or delete files. All v1 manifests track data files.
+ ManifestContent() ManifestContent
+ // SnapshotID is the ID of the snapshot where this manifest file
+ // was added.
+ SnapshotID() int64
+ // AddedDataFiles returns the number of entries in the manifest that
+ // have the status of EntryStatusADDED.
+ AddedDataFiles() int32
+ // ExistingDataFiles returns the number of entries in the manifest
+ // which have the status of EntryStatusEXISTING.
+ ExistingDataFiles() int32
+ // DeletedDataFiles returns the number of entries in the manifest
+ // which have the status of EntryStatusDELETED.
+ DeletedDataFiles() int32
+ // AddedRows returns the number of rows in all files of the manifest
+ // that have status EntryStatusADDED.
+ AddedRows() int64
+ // ExistingRows returns the number of rows in all files of the manifest
+ // which have status EntryStatusEXISTING.
+ ExistingRows() int64
+ // DeletedRows returns the number of rows in all files of the manifest
+ // which have status EntryStatusDELETED.
+ DeletedRows() int64
+ // SequenceNum returns the sequence number when this manifest was
+ // added to the table. Will be 0 for v1 manifest lists.
+ SequenceNum() int64
+ // MinSequenceNum is the minimum data sequence number of all live data
+ // or delete files in the manifest. Will be 0 for v1 manifest lists.
+ MinSequenceNum() int64
+ // Metadata returns implementation-specific key metadata for encryption
+ // if it exists in the manifest list.
+ Metadata() []byte
+ // PartitionList returns a list of field summaries for each partition
+ // field in the spec. Each field in the list corresponds to a field in
+ // the manifest file's partition spec.
+ PartitionList() []fieldSummary
+
+ // HasAddedFiles returns true if AddedDataFiles > 0 or if it was null.
+ HasAddedFiles() bool
+ // HasExistingFiles returns true if ExistingDataFiles > 0 or if it was null.
+ HasExistingFiles() bool
+ // FetchEntries reads the manifest list file to fetch the list of
+ // manifest entries using the provided file system IO interface.
+ // If discardDeleted is true, entries for files containing deleted rows
+ // will be skipped.
+ FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error)
+}
+
+// ReadManifestList reads in an avro manifest list file and returns a slice
+// of manifest files or an error if one is encountered.
+func ReadManifestList(in io.Reader) ([]ManifestFile, error) {
+ dec, err := ocf.NewDecoder(in)
+ if err != nil {
+ return nil, err
+ }
+
+ out := make([]ManifestFile, 0)
+
+ for dec.HasNext() {
+ var file ManifestFile
+ if string(dec.Metadata()["format-version"]) == "2" {
+ file = &manifestFileV2{}
+ } else {
+ file = &manifestFileV1{}
+ }
+
+ if err := dec.Decode(file); err != nil {
+ return nil, err
+ }
+ out = append(out, file)
+ }
+
+ return out, dec.Error()
+}
+
+// ManifestEntryStatus defines constants for the entry status of
+// existing, added or deleted.
+type ManifestEntryStatus int8
+
+const (
+ EntryStatusEXISTING ManifestEntryStatus = 0
+ EntryStatusADDED ManifestEntryStatus = 1
+ EntryStatusDELETED ManifestEntryStatus = 2
+)
+
+// ManifestEntryContent defines constants for the type of file contents
+// in the file entries. Data, Position based deletes and equality based
+// deletes.
+type ManifestEntryContent int8
+
+const (
+ EntryContentData ManifestEntryContent = 0
+ EntryContentPosDeletes ManifestEntryContent = 1
+ EntryContentEqDeletes ManifestEntryContent = 2
+)
+
+// FileFormat defines constants for the format of data files.
+type FileFormat string
+
+const (
+ AvroFile FileFormat = "AVRO"
+ OrcFile FileFormat = "ORC"
+ ParquetFile FileFormat = "PARQUET"
+)
+
+type colMap[K, V any] struct {
+ Key K `avro:"key"`
+ Value V `avro:"value"`
+}
+
+func avroColMapToMap[K comparable, V any](c *[]colMap[K, V]) map[K]V {
+ if c == nil {
+ return nil
+ }
+
+ out := make(map[K]V)
+ for _, data := range *c {
+ out[data.Key] = data.Value
+ }
+ return out
+}
+
+type dataFile struct {
+ Content ManifestEntryContent `avro:"content"`
+ Path string `avro:"file_path"`
+ Format FileFormat `avro:"file_format"`
+ PartitionData map[string]any `avro:"partition"`
+ RecordCount int64 `avro:"record_count"`
+ FileSize int64 `avro:"file_size_in_bytes"`
+ BlockSizeInBytes int64 `avro:"block_size_in_bytes"`
+ ColSizes *[]colMap[int, int64] `avro:"column_sizes"`
+ ValCounts *[]colMap[int, int64] `avro:"value_counts"`
+ NullCounts *[]colMap[int, int64] `avro:"null_value_counts"`
+ NaNCounts *[]colMap[int, int64] `avro:"nan_value_counts"`
+ DistinctCounts *[]colMap[int, int64] `avro:"distinct_counts"`
+ LowerBounds *[]colMap[int, []byte] `avro:"lower_bounds"`
+ UpperBounds *[]colMap[int, []byte] `avro:"upper_bounds"`
+ KeyMetadata *[]byte `avro:"key_metadata"`
+ SplitOffsets *[]int64 `avro:"split_offsets"`
+ EqualityIDs *[]int `avro:"equality_ids"`
+ SortOrder *int `avro:"sort_order_id"`
+
+ colSizeMap map[int]int64
+ valCntMap map[int]int64
+ nullCntMap map[int]int64
+ nanCntMap map[int]int64
+ distinctCntMap map[int]int64
+ lowerBoundMap map[int][]byte
+ upperBoundMap map[int][]byte
+
+ initMaps sync.Once
+}
+
+func (d *dataFile) initializeMapData() {
+ d.initMaps.Do(func() {
+ d.colSizeMap = avroColMapToMap(d.ColSizes)
+ d.valCntMap = avroColMapToMap(d.ValCounts)
+ d.nullCntMap = avroColMapToMap(d.NullCounts)
+ d.nanCntMap = avroColMapToMap(d.NaNCounts)
+ d.distinctCntMap = avroColMapToMap(d.DistinctCounts)
+ d.lowerBoundMap = avroColMapToMap(d.LowerBounds)
+ d.upperBoundMap = avroColMapToMap(d.UpperBounds)
+ })
+}
+
+func (d *dataFile) ContentType() ManifestEntryContent { return d.Content }
+func (d *dataFile) FilePath() string { return d.Path }
+func (d *dataFile) FileFormat() FileFormat { return d.Format }
+func (d *dataFile) Partition() map[string]any { return d.PartitionData }
+func (d *dataFile) Count() int64 { return d.RecordCount }
+func (d *dataFile) FileSizeBytes() int64 { return d.FileSize }
+
+func (d *dataFile) ColumnSizes() map[int]int64 {
+ d.initializeMapData()
+ return d.colSizeMap
+}
+
+func (d *dataFile) ValueCounts() map[int]int64 {
+ d.initializeMapData()
+ return d.valCntMap
+}
+
+func (d *dataFile) NullValueCounts() map[int]int64 {
+ d.initializeMapData()
+ return d.nullCntMap
+}
+
+func (d *dataFile) NaNValueCounts() map[int]int64 {
+ d.initializeMapData()
+ return d.nanCntMap
+}
+
+func (d *dataFile) DistinctValueCounts() map[int]int64 {
+ d.initializeMapData()
+ return d.distinctCntMap
+}
+
+func (d *dataFile) LowerBoundValues() map[int][]byte {
+ d.initializeMapData()
+ return d.lowerBoundMap
+}
+
+func (d *dataFile) UpperBoundValues() map[int][]byte {
+ d.initializeMapData()
+ return d.upperBoundMap
+}
+
+func (d *dataFile) MetadataKey() []byte {
+ if d.KeyMetadata == nil {
+ return nil
+ }
+ return *d.KeyMetadata
+}
+
+func (d *dataFile) Splits() []int64 {
+ if d.SplitOffsets == nil {
+ return nil
+ }
+ return *d.SplitOffsets
+}
+
+func (d *dataFile) EqualityFieldIDs() []int {
+ if d.EqualityIDs == nil {
+ return nil
+ }
+ return d.EqualityFieldIDs()
+}
+
+func (d *dataFile) SortOrderID() *int { return d.SortOrder }
+
+type manifestEntryV1 struct {
+ EntryStatus ManifestEntryStatus `avro:"status"`
+ Snapshot int64 `avro:"snapshot_id"`
+ SeqNum *int64
+ FileSeqNum *int64
+ Data dataFile `avro:"data_file"`
+}
+
+func (m *manifestEntryV1) inheritSeqNum(manifest ManifestFile) {}
+
+func (m *manifestEntryV1) Status() ManifestEntryStatus { return m.EntryStatus }
+func (m *manifestEntryV1) SnapshotID() int64 { return m.Snapshot }
+
+func (m *manifestEntryV1) SequenceNum() int64 {
+ if m.SeqNum == nil {
+ return 0
+ }
+ return *m.SeqNum
+}
+
+func (m *manifestEntryV1) FileSequenceNum() *int64 {
+ return m.FileSeqNum
+}
+
+func (m *manifestEntryV1) DataFile() DataFile { return &m.Data }
+
+type manifestEntryV2 struct {
+ EntryStatus ManifestEntryStatus `avro:"status"`
+ Snapshot *int64 `avro:"snapshot_id"`
+ SeqNum *int64 `avro:"sequence_number"`
+ FileSeqNum *int64 `avro:"file_sequence_number"`
+ Data dataFile `avro:"data_file"`
+}
+
+func (m *manifestEntryV2) inheritSeqNum(manifest ManifestFile) {
+ if m.Snapshot == nil {
+ snap := manifest.SnapshotID()
+ m.Snapshot = &snap
+ }
+
+ manifestSequenceNum := manifest.SequenceNum()
+ if m.SeqNum == nil && (manifestSequenceNum == 0 || m.EntryStatus == EntryStatusADDED) {
+ m.SeqNum = &manifestSequenceNum
+ }
+
+ if m.FileSeqNum == nil && (manifestSequenceNum == 0 || m.EntryStatus == EntryStatusADDED) {
+ m.FileSeqNum = &manifestSequenceNum
+ }
+}
+
+func (m *manifestEntryV2) Status() ManifestEntryStatus { return m.EntryStatus }
+func (m *manifestEntryV2) SnapshotID() int64 {
+ if m.Snapshot == nil {
+ return 0
+ }
+ return *m.Snapshot
+}
+
+func (m *manifestEntryV2) SequenceNum() int64 {
+ if m.SeqNum == nil {
+ return 0
+ }
+ return *m.SeqNum
+}
+
+func (m *manifestEntryV2) FileSequenceNum() *int64 {
+ return m.FileSeqNum
+}
+
+func (m *manifestEntryV2) DataFile() DataFile { return &m.Data }
+
+// DataFile is the interface for reading the information about a
+// given data file indicated by an entry in a manifest list.
+type DataFile interface {
+ // ContentType is the type of the content stored by the data file,
+ // either Data, Equality deletes, or Position deletes. All v1 files
+ // are Data files.
+ ContentType() ManifestEntryContent
+ // FilePath is the full URI for the file, complete with FS scheme.
+ FilePath() string
+ // FileFormat is the format of the data file, AVRO, Orc, or Parquet.
+ FileFormat() FileFormat
+ // Partition returns a mapping of field name to partition value for
+ // each of the partition spec's fields.
+ Partition() map[string]any
+ // Count returns the number of records in this file.
+ Count() int64
+ // FileSizeBytes is the total file size in bytes.
+ FileSizeBytes() int64
+ // ColumnSizes is a mapping from column id to the total size on disk
+ // of all regions that store the column. Does not include bytes
+ // necessary to read other columns, like footers. Map will be nil for
+ // row-oriented formats (avro).
+ ColumnSizes() map[int]int64
+ // ValueCounts is a mapping from column id to the number of values
+ // in the column, including null and NaN values.
+ ValueCounts() map[int]int64
+ // NullValueCounts is a mapping from column id to the number of
+ // null values in the column.
+ NullValueCounts() map[int]int64
+ // NaNValueCounts is a mapping from column id to the number of NaN
+ // values in the column.
+ NaNValueCounts() map[int]int64
+ // DistictValueCounts is a mapping from column id to the number of
+ // distinct values in the column. Distinct counts must be derived
+ // using values in the file by counting or using sketches, but not
+ // using methods like merging existing distinct counts.
+ DistinctValueCounts() map[int]int64
+ // LowerBoundValues is a mapping from column id to the lower bounded
+ // value of the column, serialized as binary. Each value in the column
+ // must be less than or requal to all non-null, non-NaN values in the
+ // column for the file.
+ LowerBoundValues() map[int][]byte
+ // UpperBoundValues is a mapping from column id to the upper bounded
+ // value of the column, serialized as binary. Each value in the column
+ // must be greater than or equal to all non-null, non-NaN values in
+ // the column for the file.
+ UpperBoundValues() map[int][]byte
+ // MetadataKey is implementation-specific key metadata for encryption.
+ MetadataKey() []byte
Review Comment:
should be KeyMetadata()
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] nastra commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1325905106
##########
io/io.go:
##########
@@ -0,0 +1,327 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package io
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "io/fs"
+ "net/http"
+ "net/url"
+ "os"
+ "strings"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
+ "github.com/aws/aws-sdk-go-v2/config"
+ "github.com/aws/aws-sdk-go-v2/credentials"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+ "github.com/wolfeidau/s3iofs"
+)
+
+const (
+ S3RegionName = "s3.region-name"
+ S3SessionToken = "s3.session-token"
+ S3SecretAccessKey = "s3.secret-access-key"
+ S3AccessKeyID = "s3.access-key-id"
+ S3EndpointURL = "s3.region"
Review Comment:
```suggestion
S3EndpointURL = "s3.endpoint"
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] zeroshade commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1329012208
##########
manifest.go:
##########
@@ -0,0 +1,655 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package iceberg
+
+import (
+ "io"
+ "sync"
+
+ iceio "github.com/apache/iceberg-go/io"
+
+ "github.com/hamba/avro/v2/ocf"
+)
+
+// ManifestContent indicates the type of data inside of the files
+// described by a manifest. This will indicate whether the data files
+// contain active data or deleted rows.
+type ManifestContent int32
+
+const (
+ ManifestContentData ManifestContent = 0
+ ManifestContentDeletes ManifestContent = 1
+)
+
+type fieldSummary struct {
+ ContainsNull bool `avro:"contains_null"`
+ ContainsNaN *bool `avro:"contains_nan"`
+ LowerBound *[]byte `avro:"lower_bound"`
+ UpperBound *[]byte `avro:"upper_bound"`
+}
+
+type manifestFileV1 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ AddedSnapshotID *int64 `avro:"added_snapshot_id"`
+ AddedFilesCount *int32 `avro:"added_data_files_count"`
+ ExistingFilesCount *int32 `avro:"existing_data_files_count"`
+ DeletedFilesCount *int32 `avro:"deleted_data_files_count"`
+ AddedRowsCount *int64 `avro:"added_rows_count"`
+ ExistingRowsCount *int64 `avro:"existing_rows_count"`
+ DeletedRowsCount *int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV1) Version() int { return 1 }
+func (m *manifestFileV1) FilePath() string { return m.Path }
+func (m *manifestFileV1) Length() int64 { return m.Len }
+func (m *manifestFileV1) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV1) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV1) SnapshotID() int64 {
+ if m.AddedSnapshotID == nil {
+ return 0
+ }
+ return *m.AddedSnapshotID
+}
+
+func (m *manifestFileV1) AddedDataFiles() int32 {
+ if m.AddedFilesCount == nil {
+ return 0
+ }
+ return *m.AddedFilesCount
+}
+
+func (m *manifestFileV1) ExistingDataFiles() int32 {
+ if m.ExistingFilesCount == nil {
+ return 0
+ }
+ return *m.ExistingFilesCount
+}
+
+func (m *manifestFileV1) DeletedDataFiles() int32 {
+ if m.DeletedFilesCount == nil {
+ return 0
+ }
+ return *m.DeletedFilesCount
+}
+
+func (m *manifestFileV1) AddedRows() int64 {
+ if m.AddedRowsCount == nil {
+ return 0
+ }
+ return *m.AddedRowsCount
+}
+
+func (m *manifestFileV1) ExistingRows() int64 {
+ if m.ExistingRowsCount == nil {
+ return 0
+ }
+ return *m.ExistingRowsCount
+}
+
+func (m *manifestFileV1) DeletedRows() int64 {
+ if m.DeletedRowsCount == nil {
+ return 0
+ }
+ return *m.DeletedRowsCount
+}
+
+func (m *manifestFileV1) HasAddedFiles() bool {
+ return m.AddedFilesCount == nil || *m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV1) HasExistingFiles() bool {
+ return m.ExistingFilesCount == nil || *m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV1) SequenceNum() int64 { return 0 }
+func (m *manifestFileV1) MinSequenceNum() int64 { return 0 }
+func (m *manifestFileV1) Metadata() []byte { return m.KeyMetadata }
+func (m *manifestFileV1) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV1) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+type manifestFileV2 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ SeqNumber int64 `avro:"sequence_number"`
+ MinSeqNumber int64 `avro:"min_sequence_number"`
+ AddedSnapshotID int64 `avro:"added_snapshot_id"`
+ AddedFilesCount int32 `avro:"added_files_count"`
+ ExistingFilesCount int32 `avro:"existing_files_count"`
+ DeletedFilesCount int32 `avro:"deleted_files_count"`
+ AddedRowsCount int64 `avro:"added_rows_count"`
+ ExistingRowsCount int64 `avro:"existing_rows_count"`
+ DeletedRowsCount int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV2) Version() int { return 2 }
+
+func (m *manifestFileV2) FilePath() string { return m.Path }
+func (m *manifestFileV2) Length() int64 { return m.Len }
+func (m *manifestFileV2) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV2) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV2) SnapshotID() int64 {
+ return m.AddedSnapshotID
+}
+
+func (m *manifestFileV2) AddedDataFiles() int32 {
+ return m.AddedFilesCount
+}
+
+func (m *manifestFileV2) ExistingDataFiles() int32 {
+ return m.ExistingFilesCount
+}
+
+func (m *manifestFileV2) DeletedDataFiles() int32 {
+ return m.DeletedFilesCount
+}
+
+func (m *manifestFileV2) AddedRows() int64 {
+ return m.AddedRowsCount
+}
+
+func (m *manifestFileV2) ExistingRows() int64 {
+ return m.ExistingRowsCount
+}
+
+func (m *manifestFileV2) DeletedRows() int64 {
+ return m.DeletedRowsCount
+}
+
+func (m *manifestFileV2) SequenceNum() int64 { return m.SeqNumber }
+func (m *manifestFileV2) MinSequenceNum() int64 { return m.MinSeqNumber }
+func (m *manifestFileV2) Metadata() []byte { return m.KeyMetadata }
+
+func (m *manifestFileV2) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV2) HasAddedFiles() bool {
+ return m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV2) HasExistingFiles() bool {
+ return m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV2) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+func fetchManifestEntries(m ManifestFile, fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ f, err := fs.Open(m.FilePath())
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+
+ dec, err := ocf.NewDecoder(f)
+ if err != nil {
+ return nil, err
+ }
+
+ metadata := dec.Metadata()
+ isVer1 := true
+ if string(metadata["format-version"]) == "2" {
+ isVer1 = false
+ }
+
+ results := make([]ManifestEntry, 0)
+ for dec.HasNext() {
+ var tmp ManifestEntry
+ if isVer1 {
+ tmp = &manifestEntryV1{}
+ } else {
+ tmp = &manifestEntryV2{}
+ }
+
+ if err := dec.Decode(tmp); err != nil {
+ return nil, err
+ }
+
+ if !discardDeleted || tmp.Status() != EntryStatusDELETED {
+ tmp.inheritSeqNum(m)
+ results = append(results, tmp)
+ }
+ }
+
+ return results, dec.Error()
+}
+
+// ManifestFile is the interface which covers both V1 and V2 manifest files.
+type ManifestFile interface {
+ // Version returns the version number of this manifest file.
+ // It should be 1 or 2.
+ Version() int
+ // FilePath is the location URI of this manifest file.
+ FilePath() string
+ // Length is the length in bytes of the manifest file.
+ Length() int64
+ // PartitionID is the ID of the partition spec used to write
+ // this manifest. It must be listed in the table metadata
+ // partition-specs.
+ PartitionID() int32
+ // ManifestContent is the type of files tracked by this manifest,
+ // either data or delete files. All v1 manifests track data files.
+ ManifestContent() ManifestContent
+ // SnapshotID is the ID of the snapshot where this manifest file
+ // was added.
+ SnapshotID() int64
+ // AddedDataFiles returns the number of entries in the manifest that
+ // have the status of EntryStatusADDED.
+ AddedDataFiles() int32
+ // ExistingDataFiles returns the number of entries in the manifest
+ // which have the status of EntryStatusEXISTING.
+ ExistingDataFiles() int32
+ // DeletedDataFiles returns the number of entries in the manifest
+ // which have the status of EntryStatusDELETED.
+ DeletedDataFiles() int32
+ // AddedRows returns the number of rows in all files of the manifest
+ // that have status EntryStatusADDED.
+ AddedRows() int64
+ // ExistingRows returns the number of rows in all files of the manifest
+ // which have status EntryStatusEXISTING.
+ ExistingRows() int64
+ // DeletedRows returns the number of rows in all files of the manifest
+ // which have status EntryStatusDELETED.
+ DeletedRows() int64
+ // SequenceNum returns the sequence number when this manifest was
+ // added to the table. Will be 0 for v1 manifest lists.
+ SequenceNum() int64
+ // MinSequenceNum is the minimum data sequence number of all live data
+ // or delete files in the manifest. Will be 0 for v1 manifest lists.
+ MinSequenceNum() int64
+ // Metadata returns implementation-specific key metadata for encryption
+ // if it exists in the manifest list.
+ Metadata() []byte
+ // PartitionList returns a list of field summaries for each partition
+ // field in the spec. Each field in the list corresponds to a field in
+ // the manifest file's partition spec.
+ PartitionList() []fieldSummary
+
+ // HasAddedFiles returns true if AddedDataFiles > 0 or if it was null.
+ HasAddedFiles() bool
+ // HasExistingFiles returns true if ExistingDataFiles > 0 or if it was null.
+ HasExistingFiles() bool
+ // FetchEntries reads the manifest list file to fetch the list of
+ // manifest entries using the provided file system IO interface.
+ // If discardDeleted is true, entries for files containing deleted rows
+ // will be skipped.
+ FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error)
+}
+
+// ReadManifestList reads in an avro manifest list file and returns a slice
+// of manifest files or an error if one is encountered.
+func ReadManifestList(in io.Reader) ([]ManifestFile, error) {
+ dec, err := ocf.NewDecoder(in)
+ if err != nil {
+ return nil, err
+ }
+
+ out := make([]ManifestFile, 0)
+
+ for dec.HasNext() {
+ var file ManifestFile
+ if string(dec.Metadata()["format-version"]) == "2" {
+ file = &manifestFileV2{}
+ } else {
+ file = &manifestFileV1{}
+ }
+
+ if err := dec.Decode(file); err != nil {
+ return nil, err
+ }
+ out = append(out, file)
+ }
+
+ return out, dec.Error()
+}
+
+// ManifestEntryStatus defines constants for the entry status of
+// existing, added or deleted.
+type ManifestEntryStatus int8
+
+const (
+ EntryStatusEXISTING ManifestEntryStatus = 0
+ EntryStatusADDED ManifestEntryStatus = 1
+ EntryStatusDELETED ManifestEntryStatus = 2
+)
+
+// ManifestEntryContent defines constants for the type of file contents
+// in the file entries. Data, Position based deletes and equality based
+// deletes.
+type ManifestEntryContent int8
+
+const (
+ EntryContentData ManifestEntryContent = 0
+ EntryContentPosDeletes ManifestEntryContent = 1
+ EntryContentEqDeletes ManifestEntryContent = 2
+)
+
+// FileFormat defines constants for the format of data files.
+type FileFormat string
+
+const (
+ AvroFile FileFormat = "AVRO"
+ OrcFile FileFormat = "ORC"
+ ParquetFile FileFormat = "PARQUET"
+)
+
+type colMap[K, V any] struct {
+ Key K `avro:"key"`
+ Value V `avro:"value"`
+}
+
+func avroColMapToMap[K comparable, V any](c *[]colMap[K, V]) map[K]V {
+ if c == nil {
+ return nil
+ }
+
+ out := make(map[K]V)
+ for _, data := range *c {
+ out[data.Key] = data.Value
+ }
+ return out
+}
+
+type dataFile struct {
+ Content ManifestEntryContent `avro:"content"`
+ Path string `avro:"file_path"`
+ Format FileFormat `avro:"file_format"`
+ PartitionData map[string]any `avro:"partition"`
+ RecordCount int64 `avro:"record_count"`
+ FileSize int64 `avro:"file_size_in_bytes"`
+ BlockSizeInBytes int64 `avro:"block_size_in_bytes"`
+ ColSizes *[]colMap[int, int64] `avro:"column_sizes"`
+ ValCounts *[]colMap[int, int64] `avro:"value_counts"`
+ NullCounts *[]colMap[int, int64] `avro:"null_value_counts"`
+ NaNCounts *[]colMap[int, int64] `avro:"nan_value_counts"`
+ DistinctCounts *[]colMap[int, int64] `avro:"distinct_counts"`
+ LowerBounds *[]colMap[int, []byte] `avro:"lower_bounds"`
+ UpperBounds *[]colMap[int, []byte] `avro:"upper_bounds"`
+ KeyMetadata *[]byte `avro:"key_metadata"`
+ SplitOffsets *[]int64 `avro:"split_offsets"`
+ EqualityIDs *[]int `avro:"equality_ids"`
+ SortOrder *int `avro:"sort_order_id"`
+
+ colSizeMap map[int]int64
+ valCntMap map[int]int64
+ nullCntMap map[int]int64
+ nanCntMap map[int]int64
+ distinctCntMap map[int]int64
+ lowerBoundMap map[int][]byte
+ upperBoundMap map[int][]byte
+
+ initMaps sync.Once
+}
+
+func (d *dataFile) initializeMapData() {
+ d.initMaps.Do(func() {
+ d.colSizeMap = avroColMapToMap(d.ColSizes)
+ d.valCntMap = avroColMapToMap(d.ValCounts)
+ d.nullCntMap = avroColMapToMap(d.NullCounts)
+ d.nanCntMap = avroColMapToMap(d.NaNCounts)
+ d.distinctCntMap = avroColMapToMap(d.DistinctCounts)
+ d.lowerBoundMap = avroColMapToMap(d.LowerBounds)
+ d.upperBoundMap = avroColMapToMap(d.UpperBounds)
+ })
+}
+
+func (d *dataFile) ContentType() ManifestEntryContent { return d.Content }
+func (d *dataFile) FilePath() string { return d.Path }
+func (d *dataFile) FileFormat() FileFormat { return d.Format }
+func (d *dataFile) Partition() map[string]any { return d.PartitionData }
+func (d *dataFile) Count() int64 { return d.RecordCount }
+func (d *dataFile) FileSizeBytes() int64 { return d.FileSize }
+
+func (d *dataFile) ColumnSizes() map[int]int64 {
+ d.initializeMapData()
+ return d.colSizeMap
+}
+
+func (d *dataFile) ValueCounts() map[int]int64 {
+ d.initializeMapData()
+ return d.valCntMap
+}
+
+func (d *dataFile) NullValueCounts() map[int]int64 {
+ d.initializeMapData()
+ return d.nullCntMap
+}
+
+func (d *dataFile) NaNValueCounts() map[int]int64 {
+ d.initializeMapData()
+ return d.nanCntMap
+}
+
+func (d *dataFile) DistinctValueCounts() map[int]int64 {
+ d.initializeMapData()
+ return d.distinctCntMap
+}
+
+func (d *dataFile) LowerBoundValues() map[int][]byte {
+ d.initializeMapData()
+ return d.lowerBoundMap
+}
+
+func (d *dataFile) UpperBoundValues() map[int][]byte {
+ d.initializeMapData()
+ return d.upperBoundMap
+}
+
+func (d *dataFile) MetadataKey() []byte {
+ if d.KeyMetadata == nil {
+ return nil
+ }
+ return *d.KeyMetadata
+}
+
+func (d *dataFile) Splits() []int64 {
+ if d.SplitOffsets == nil {
+ return nil
+ }
+ return *d.SplitOffsets
+}
+
+func (d *dataFile) EqualityFieldIDs() []int {
+ if d.EqualityIDs == nil {
+ return nil
+ }
+ return d.EqualityFieldIDs()
+}
+
+func (d *dataFile) SortOrderID() *int { return d.SortOrder }
+
+type manifestEntryV1 struct {
+ EntryStatus ManifestEntryStatus `avro:"status"`
+ Snapshot int64 `avro:"snapshot_id"`
+ SeqNum *int64
+ FileSeqNum *int64
+ Data dataFile `avro:"data_file"`
+}
+
+func (m *manifestEntryV1) inheritSeqNum(manifest ManifestFile) {}
+
+func (m *manifestEntryV1) Status() ManifestEntryStatus { return m.EntryStatus }
+func (m *manifestEntryV1) SnapshotID() int64 { return m.Snapshot }
+
+func (m *manifestEntryV1) SequenceNum() int64 {
+ if m.SeqNum == nil {
+ return 0
+ }
+ return *m.SeqNum
+}
+
+func (m *manifestEntryV1) FileSequenceNum() *int64 {
+ return m.FileSeqNum
+}
+
+func (m *manifestEntryV1) DataFile() DataFile { return &m.Data }
+
+type manifestEntryV2 struct {
+ EntryStatus ManifestEntryStatus `avro:"status"`
+ Snapshot *int64 `avro:"snapshot_id"`
+ SeqNum *int64 `avro:"sequence_number"`
+ FileSeqNum *int64 `avro:"file_sequence_number"`
+ Data dataFile `avro:"data_file"`
+}
+
+func (m *manifestEntryV2) inheritSeqNum(manifest ManifestFile) {
+ if m.Snapshot == nil {
+ snap := manifest.SnapshotID()
+ m.Snapshot = &snap
+ }
+
+ manifestSequenceNum := manifest.SequenceNum()
+ if m.SeqNum == nil && (manifestSequenceNum == 0 || m.EntryStatus == EntryStatusADDED) {
+ m.SeqNum = &manifestSequenceNum
+ }
+
+ if m.FileSeqNum == nil && (manifestSequenceNum == 0 || m.EntryStatus == EntryStatusADDED) {
+ m.FileSeqNum = &manifestSequenceNum
+ }
+}
+
+func (m *manifestEntryV2) Status() ManifestEntryStatus { return m.EntryStatus }
+func (m *manifestEntryV2) SnapshotID() int64 {
+ if m.Snapshot == nil {
+ return 0
+ }
+ return *m.Snapshot
+}
+
+func (m *manifestEntryV2) SequenceNum() int64 {
+ if m.SeqNum == nil {
+ return 0
+ }
+ return *m.SeqNum
+}
+
+func (m *manifestEntryV2) FileSequenceNum() *int64 {
+ return m.FileSeqNum
+}
+
+func (m *manifestEntryV2) DataFile() DataFile { return &m.Data }
+
+// DataFile is the interface for reading the information about a
+// given data file indicated by an entry in a manifest list.
+type DataFile interface {
+ // ContentType is the type of the content stored by the data file,
+ // either Data, Equality deletes, or Position deletes. All v1 files
+ // are Data files.
+ ContentType() ManifestEntryContent
+ // FilePath is the full URI for the file, complete with FS scheme.
+ FilePath() string
+ // FileFormat is the format of the data file, AVRO, Orc, or Parquet.
+ FileFormat() FileFormat
+ // Partition returns a mapping of field name to partition value for
+ // each of the partition spec's fields.
+ Partition() map[string]any
+ // Count returns the number of records in this file.
+ Count() int64
+ // FileSizeBytes is the total file size in bytes.
+ FileSizeBytes() int64
+ // ColumnSizes is a mapping from column id to the total size on disk
+ // of all regions that store the column. Does not include bytes
+ // necessary to read other columns, like footers. Map will be nil for
+ // row-oriented formats (avro).
+ ColumnSizes() map[int]int64
+ // ValueCounts is a mapping from column id to the number of values
+ // in the column, including null and NaN values.
+ ValueCounts() map[int]int64
+ // NullValueCounts is a mapping from column id to the number of
+ // null values in the column.
+ NullValueCounts() map[int]int64
+ // NaNValueCounts is a mapping from column id to the number of NaN
+ // values in the column.
+ NaNValueCounts() map[int]int64
+ // DistictValueCounts is a mapping from column id to the number of
+ // distinct values in the column. Distinct counts must be derived
+ // using values in the file by counting or using sketches, but not
+ // using methods like merging existing distinct counts.
+ DistinctValueCounts() map[int]int64
+ // LowerBoundValues is a mapping from column id to the lower bounded
+ // value of the column, serialized as binary. Each value in the column
+ // must be less than or requal to all non-null, non-NaN values in the
+ // column for the file.
+ LowerBoundValues() map[int][]byte
+ // UpperBoundValues is a mapping from column id to the upper bounded
+ // value of the column, serialized as binary. Each value in the column
+ // must be greater than or equal to all non-null, non-NaN values in
+ // the column for the file.
+ UpperBoundValues() map[int][]byte
+ // MetadataKey is implementation-specific key metadata for encryption.
+ MetadataKey() []byte
+ // Splits are the split offsets for the data file. For example,
+ // all row group offsets in a Parquet file. Must be sorted ascending.
+ Splits() []int64
+ // EqualityFieldIDs are used to determine row equality in equality
+ // delete files. It is required when the content type is
+ // EntryContentEqDeletes.
+ EqualityFieldIDs() []int
+ // SortOrderID returns the id representing the sort order for this
+ // file, or nil if there is no sort order.
+ SortOrderID() *int
+}
Review Comment:
Hmm, i'm reading through the Java code and the best I can see is that the `ContentFile` interface wraps around the entire `ManifestEntry` objects and essentially surfaces the individual fields of the data file to the same level as the manifest entry information (which includes the `DataSequenceNumber` and `FileSequenceNumber`. But if you look at the spec (https://iceberg.apache.org/spec/#manifests) those values exist in the `manifest_entry` struct, not the `data_file` struct, which is why they aren't exposed here at this level.
Looking at the Builder for data files in the Java code (https://github.com/apache/iceberg/blob/d6bc248adb67de74e31dcb9c0af43fef68853d59/core/src/main/java/org/apache/iceberg/DataFiles.java) it also doesn't allow setting the sequence number values down there. You can see both the Data Sequence number and File Sequence Number in the `ManifestEntry` interface in the Go code.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] nastra commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1326996787
##########
manifest.go:
##########
@@ -0,0 +1,655 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package iceberg
+
+import (
+ "io"
+ "sync"
+
+ iceio "github.com/apache/iceberg-go/io"
+
+ "github.com/hamba/avro/v2/ocf"
+)
+
+// ManifestContent indicates the type of data inside of the files
+// described by a manifest. This will indicate whether the data files
+// contain active data or deleted rows.
+type ManifestContent int32
+
+const (
+ ManifestContentData ManifestContent = 0
+ ManifestContentDeletes ManifestContent = 1
+)
+
+type fieldSummary struct {
+ ContainsNull bool `avro:"contains_null"`
+ ContainsNaN *bool `avro:"contains_nan"`
+ LowerBound *[]byte `avro:"lower_bound"`
+ UpperBound *[]byte `avro:"upper_bound"`
+}
+
+type manifestFileV1 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ AddedSnapshotID *int64 `avro:"added_snapshot_id"`
+ AddedFilesCount *int32 `avro:"added_data_files_count"`
+ ExistingFilesCount *int32 `avro:"existing_data_files_count"`
+ DeletedFilesCount *int32 `avro:"deleted_data_files_count"`
+ AddedRowsCount *int64 `avro:"added_rows_count"`
+ ExistingRowsCount *int64 `avro:"existing_rows_count"`
+ DeletedRowsCount *int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV1) Version() int { return 1 }
+func (m *manifestFileV1) FilePath() string { return m.Path }
+func (m *manifestFileV1) Length() int64 { return m.Len }
+func (m *manifestFileV1) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV1) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV1) SnapshotID() int64 {
+ if m.AddedSnapshotID == nil {
+ return 0
+ }
+ return *m.AddedSnapshotID
+}
+
+func (m *manifestFileV1) AddedDataFiles() int32 {
+ if m.AddedFilesCount == nil {
+ return 0
+ }
+ return *m.AddedFilesCount
+}
+
+func (m *manifestFileV1) ExistingDataFiles() int32 {
+ if m.ExistingFilesCount == nil {
+ return 0
+ }
+ return *m.ExistingFilesCount
+}
+
+func (m *manifestFileV1) DeletedDataFiles() int32 {
+ if m.DeletedFilesCount == nil {
+ return 0
+ }
+ return *m.DeletedFilesCount
+}
+
+func (m *manifestFileV1) AddedRows() int64 {
+ if m.AddedRowsCount == nil {
+ return 0
+ }
+ return *m.AddedRowsCount
+}
+
+func (m *manifestFileV1) ExistingRows() int64 {
+ if m.ExistingRowsCount == nil {
+ return 0
+ }
+ return *m.ExistingRowsCount
+}
+
+func (m *manifestFileV1) DeletedRows() int64 {
+ if m.DeletedRowsCount == nil {
+ return 0
+ }
+ return *m.DeletedRowsCount
+}
+
+func (m *manifestFileV1) HasAddedFiles() bool {
+ return m.AddedFilesCount == nil || *m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV1) HasExistingFiles() bool {
+ return m.ExistingFilesCount == nil || *m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV1) SequenceNum() int64 { return 0 }
+func (m *manifestFileV1) MinSequenceNum() int64 { return 0 }
+func (m *manifestFileV1) Metadata() []byte { return m.KeyMetadata }
+func (m *manifestFileV1) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV1) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+type manifestFileV2 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ SeqNumber int64 `avro:"sequence_number"`
+ MinSeqNumber int64 `avro:"min_sequence_number"`
+ AddedSnapshotID int64 `avro:"added_snapshot_id"`
+ AddedFilesCount int32 `avro:"added_files_count"`
+ ExistingFilesCount int32 `avro:"existing_files_count"`
+ DeletedFilesCount int32 `avro:"deleted_files_count"`
+ AddedRowsCount int64 `avro:"added_rows_count"`
+ ExistingRowsCount int64 `avro:"existing_rows_count"`
+ DeletedRowsCount int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV2) Version() int { return 2 }
+
+func (m *manifestFileV2) FilePath() string { return m.Path }
+func (m *manifestFileV2) Length() int64 { return m.Len }
+func (m *manifestFileV2) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV2) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV2) SnapshotID() int64 {
+ return m.AddedSnapshotID
+}
+
+func (m *manifestFileV2) AddedDataFiles() int32 {
+ return m.AddedFilesCount
+}
+
+func (m *manifestFileV2) ExistingDataFiles() int32 {
+ return m.ExistingFilesCount
+}
+
+func (m *manifestFileV2) DeletedDataFiles() int32 {
+ return m.DeletedFilesCount
+}
+
+func (m *manifestFileV2) AddedRows() int64 {
+ return m.AddedRowsCount
+}
+
+func (m *manifestFileV2) ExistingRows() int64 {
+ return m.ExistingRowsCount
+}
+
+func (m *manifestFileV2) DeletedRows() int64 {
+ return m.DeletedRowsCount
+}
+
+func (m *manifestFileV2) SequenceNum() int64 { return m.SeqNumber }
+func (m *manifestFileV2) MinSequenceNum() int64 { return m.MinSeqNumber }
+func (m *manifestFileV2) Metadata() []byte { return m.KeyMetadata }
+
+func (m *manifestFileV2) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV2) HasAddedFiles() bool {
+ return m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV2) HasExistingFiles() bool {
+ return m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV2) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+func fetchManifestEntries(m ManifestFile, fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ f, err := fs.Open(m.FilePath())
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+
+ dec, err := ocf.NewDecoder(f)
+ if err != nil {
+ return nil, err
+ }
+
+ metadata := dec.Metadata()
+ isVer1 := true
+ if string(metadata["format-version"]) == "2" {
+ isVer1 = false
+ }
+
+ results := make([]ManifestEntry, 0)
+ for dec.HasNext() {
+ var tmp ManifestEntry
+ if isVer1 {
+ tmp = &manifestEntryV1{}
+ } else {
+ tmp = &manifestEntryV2{}
+ }
+
+ if err := dec.Decode(tmp); err != nil {
+ return nil, err
+ }
+
+ if !discardDeleted || tmp.Status() != EntryStatusDELETED {
+ tmp.inheritSeqNum(m)
+ results = append(results, tmp)
+ }
+ }
+
+ return results, dec.Error()
+}
+
+// ManifestFile is the interface which covers both V1 and V2 manifest files.
+type ManifestFile interface {
+ // Version returns the version number of this manifest file.
+ // It should be 1 or 2.
+ Version() int
+ // FilePath is the location URI of this manifest file.
+ FilePath() string
+ // Length is the length in bytes of the manifest file.
+ Length() int64
+ // PartitionID is the ID of the partition spec used to write
+ // this manifest. It must be listed in the table metadata
+ // partition-specs.
+ PartitionID() int32
+ // ManifestContent is the type of files tracked by this manifest,
+ // either data or delete files. All v1 manifests track data files.
+ ManifestContent() ManifestContent
+ // SnapshotID is the ID of the snapshot where this manifest file
+ // was added.
+ SnapshotID() int64
+ // AddedDataFiles returns the number of entries in the manifest that
+ // have the status of EntryStatusADDED.
+ AddedDataFiles() int32
+ // ExistingDataFiles returns the number of entries in the manifest
+ // which have the status of EntryStatusEXISTING.
+ ExistingDataFiles() int32
+ // DeletedDataFiles returns the number of entries in the manifest
+ // which have the status of EntryStatusDELETED.
+ DeletedDataFiles() int32
+ // AddedRows returns the number of rows in all files of the manifest
+ // that have status EntryStatusADDED.
+ AddedRows() int64
+ // ExistingRows returns the number of rows in all files of the manifest
+ // which have status EntryStatusEXISTING.
+ ExistingRows() int64
+ // DeletedRows returns the number of rows in all files of the manifest
+ // which have status EntryStatusDELETED.
+ DeletedRows() int64
+ // SequenceNum returns the sequence number when this manifest was
+ // added to the table. Will be 0 for v1 manifest lists.
+ SequenceNum() int64
+ // MinSequenceNum is the minimum data sequence number of all live data
+ // or delete files in the manifest. Will be 0 for v1 manifest lists.
+ MinSequenceNum() int64
+ // Metadata returns implementation-specific key metadata for encryption
+ // if it exists in the manifest list.
+ Metadata() []byte
+ // PartitionList returns a list of field summaries for each partition
+ // field in the spec. Each field in the list corresponds to a field in
+ // the manifest file's partition spec.
+ PartitionList() []fieldSummary
+
+ // HasAddedFiles returns true if AddedDataFiles > 0 or if it was null.
+ HasAddedFiles() bool
+ // HasExistingFiles returns true if ExistingDataFiles > 0 or if it was null.
+ HasExistingFiles() bool
+ // FetchEntries reads the manifest list file to fetch the list of
+ // manifest entries using the provided file system IO interface.
+ // If discardDeleted is true, entries for files containing deleted rows
+ // will be skipped.
+ FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error)
+}
+
+// ReadManifestList reads in an avro manifest list file and returns a slice
+// of manifest files or an error if one is encountered.
+func ReadManifestList(in io.Reader) ([]ManifestFile, error) {
+ dec, err := ocf.NewDecoder(in)
+ if err != nil {
+ return nil, err
+ }
+
+ out := make([]ManifestFile, 0)
+
+ for dec.HasNext() {
+ var file ManifestFile
+ if string(dec.Metadata()["format-version"]) == "2" {
+ file = &manifestFileV2{}
+ } else {
+ file = &manifestFileV1{}
+ }
+
+ if err := dec.Decode(file); err != nil {
+ return nil, err
+ }
+ out = append(out, file)
+ }
+
+ return out, dec.Error()
+}
+
+// ManifestEntryStatus defines constants for the entry status of
+// existing, added or deleted.
+type ManifestEntryStatus int8
+
+const (
+ EntryStatusEXISTING ManifestEntryStatus = 0
+ EntryStatusADDED ManifestEntryStatus = 1
+ EntryStatusDELETED ManifestEntryStatus = 2
+)
+
+// ManifestEntryContent defines constants for the type of file contents
+// in the file entries. Data, Position based deletes and equality based
+// deletes.
+type ManifestEntryContent int8
+
+const (
+ EntryContentData ManifestEntryContent = 0
+ EntryContentPosDeletes ManifestEntryContent = 1
+ EntryContentEqDeletes ManifestEntryContent = 2
+)
+
+// FileFormat defines constants for the format of data files.
+type FileFormat string
+
+const (
+ AvroFile FileFormat = "AVRO"
+ OrcFile FileFormat = "ORC"
+ ParquetFile FileFormat = "PARQUET"
+)
+
+type colMap[K, V any] struct {
+ Key K `avro:"key"`
+ Value V `avro:"value"`
+}
+
+func avroColMapToMap[K comparable, V any](c *[]colMap[K, V]) map[K]V {
+ if c == nil {
+ return nil
+ }
+
+ out := make(map[K]V)
+ for _, data := range *c {
+ out[data.Key] = data.Value
+ }
+ return out
+}
+
+type dataFile struct {
+ Content ManifestEntryContent `avro:"content"`
+ Path string `avro:"file_path"`
+ Format FileFormat `avro:"file_format"`
+ PartitionData map[string]any `avro:"partition"`
+ RecordCount int64 `avro:"record_count"`
+ FileSize int64 `avro:"file_size_in_bytes"`
+ BlockSizeInBytes int64 `avro:"block_size_in_bytes"`
Review Comment:
I think this shouldn't be written for V2
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat(manifests): Adding implementation of manifest files [iceberg-go]
Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1354308095
##########
manifest.go:
##########
@@ -0,0 +1,655 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package iceberg
+
+import (
+ "io"
+ "sync"
+
+ iceio "github.com/apache/iceberg-go/io"
+
+ "github.com/hamba/avro/v2/ocf"
+)
+
+// ManifestContent indicates the type of data inside of the files
+// described by a manifest. This will indicate whether the data files
+// contain active data or deleted rows.
+type ManifestContent int32
+
+const (
+ ManifestContentData ManifestContent = 0
+ ManifestContentDeletes ManifestContent = 1
+)
+
+type fieldSummary struct {
+ ContainsNull bool `avro:"contains_null"`
+ ContainsNaN *bool `avro:"contains_nan"`
+ LowerBound *[]byte `avro:"lower_bound"`
+ UpperBound *[]byte `avro:"upper_bound"`
+}
+
+type manifestFileV1 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
Review Comment:
@zeroshade just FYI that `added_snapshot_id` is mandatory after all for V1 and V2, so you might want to follow-up and make it required
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] zeroshade commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1334836935
##########
manifest.go:
##########
@@ -0,0 +1,655 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package iceberg
+
+import (
+ "io"
+ "sync"
+
+ iceio "github.com/apache/iceberg-go/io"
+
+ "github.com/hamba/avro/v2/ocf"
+)
+
+// ManifestContent indicates the type of data inside of the files
+// described by a manifest. This will indicate whether the data files
+// contain active data or deleted rows.
+type ManifestContent int32
+
+const (
+ ManifestContentData ManifestContent = 0
+ ManifestContentDeletes ManifestContent = 1
+)
+
+type fieldSummary struct {
+ ContainsNull bool `avro:"contains_null"`
+ ContainsNaN *bool `avro:"contains_nan"`
+ LowerBound *[]byte `avro:"lower_bound"`
+ UpperBound *[]byte `avro:"upper_bound"`
+}
+
+type manifestFileV1 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ AddedSnapshotID *int64 `avro:"added_snapshot_id"`
+ AddedFilesCount *int32 `avro:"added_data_files_count"`
+ ExistingFilesCount *int32 `avro:"existing_data_files_count"`
+ DeletedFilesCount *int32 `avro:"deleted_data_files_count"`
+ AddedRowsCount *int64 `avro:"added_rows_count"`
+ ExistingRowsCount *int64 `avro:"existing_rows_count"`
+ DeletedRowsCount *int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV1) Version() int { return 1 }
+func (m *manifestFileV1) FilePath() string { return m.Path }
+func (m *manifestFileV1) Length() int64 { return m.Len }
+func (m *manifestFileV1) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV1) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV1) SnapshotID() int64 {
+ if m.AddedSnapshotID == nil {
+ return 0
+ }
+ return *m.AddedSnapshotID
+}
+
+func (m *manifestFileV1) AddedDataFiles() int32 {
+ if m.AddedFilesCount == nil {
+ return 0
+ }
+ return *m.AddedFilesCount
+}
+
+func (m *manifestFileV1) ExistingDataFiles() int32 {
+ if m.ExistingFilesCount == nil {
+ return 0
+ }
+ return *m.ExistingFilesCount
+}
+
+func (m *manifestFileV1) DeletedDataFiles() int32 {
+ if m.DeletedFilesCount == nil {
+ return 0
+ }
+ return *m.DeletedFilesCount
+}
+
+func (m *manifestFileV1) AddedRows() int64 {
+ if m.AddedRowsCount == nil {
+ return 0
+ }
+ return *m.AddedRowsCount
+}
+
+func (m *manifestFileV1) ExistingRows() int64 {
+ if m.ExistingRowsCount == nil {
+ return 0
+ }
+ return *m.ExistingRowsCount
+}
+
+func (m *manifestFileV1) DeletedRows() int64 {
+ if m.DeletedRowsCount == nil {
+ return 0
+ }
+ return *m.DeletedRowsCount
+}
+
+func (m *manifestFileV1) HasAddedFiles() bool {
+ return m.AddedFilesCount == nil || *m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV1) HasExistingFiles() bool {
+ return m.ExistingFilesCount == nil || *m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV1) SequenceNum() int64 { return 0 }
+func (m *manifestFileV1) MinSequenceNum() int64 { return 0 }
+func (m *manifestFileV1) Metadata() []byte { return m.KeyMetadata }
+func (m *manifestFileV1) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV1) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+type manifestFileV2 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ SeqNumber int64 `avro:"sequence_number"`
+ MinSeqNumber int64 `avro:"min_sequence_number"`
+ AddedSnapshotID int64 `avro:"added_snapshot_id"`
+ AddedFilesCount int32 `avro:"added_files_count"`
+ ExistingFilesCount int32 `avro:"existing_files_count"`
+ DeletedFilesCount int32 `avro:"deleted_files_count"`
+ AddedRowsCount int64 `avro:"added_rows_count"`
+ ExistingRowsCount int64 `avro:"existing_rows_count"`
+ DeletedRowsCount int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV2) Version() int { return 2 }
+
+func (m *manifestFileV2) FilePath() string { return m.Path }
+func (m *manifestFileV2) Length() int64 { return m.Len }
+func (m *manifestFileV2) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV2) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV2) SnapshotID() int64 {
+ return m.AddedSnapshotID
+}
+
+func (m *manifestFileV2) AddedDataFiles() int32 {
+ return m.AddedFilesCount
+}
+
+func (m *manifestFileV2) ExistingDataFiles() int32 {
+ return m.ExistingFilesCount
+}
+
+func (m *manifestFileV2) DeletedDataFiles() int32 {
+ return m.DeletedFilesCount
+}
+
+func (m *manifestFileV2) AddedRows() int64 {
+ return m.AddedRowsCount
+}
+
+func (m *manifestFileV2) ExistingRows() int64 {
+ return m.ExistingRowsCount
+}
+
+func (m *manifestFileV2) DeletedRows() int64 {
+ return m.DeletedRowsCount
+}
+
+func (m *manifestFileV2) SequenceNum() int64 { return m.SeqNumber }
+func (m *manifestFileV2) MinSequenceNum() int64 { return m.MinSeqNumber }
+func (m *manifestFileV2) Metadata() []byte { return m.KeyMetadata }
+
+func (m *manifestFileV2) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV2) HasAddedFiles() bool {
+ return m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV2) HasExistingFiles() bool {
+ return m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV2) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+func fetchManifestEntries(m ManifestFile, fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ f, err := fs.Open(m.FilePath())
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+
+ dec, err := ocf.NewDecoder(f)
+ if err != nil {
+ return nil, err
+ }
+
+ metadata := dec.Metadata()
+ isVer1 := true
+ if string(metadata["format-version"]) == "2" {
+ isVer1 = false
+ }
+
+ results := make([]ManifestEntry, 0)
+ for dec.HasNext() {
+ var tmp ManifestEntry
+ if isVer1 {
+ tmp = &manifestEntryV1{}
+ } else {
+ tmp = &manifestEntryV2{}
+ }
+
+ if err := dec.Decode(tmp); err != nil {
+ return nil, err
+ }
+
+ if !discardDeleted || tmp.Status() != EntryStatusDELETED {
+ tmp.inheritSeqNum(m)
+ results = append(results, tmp)
+ }
+ }
+
+ return results, dec.Error()
+}
+
+// ManifestFile is the interface which covers both V1 and V2 manifest files.
+type ManifestFile interface {
+ // Version returns the version number of this manifest file.
+ // It should be 1 or 2.
+ Version() int
+ // FilePath is the location URI of this manifest file.
+ FilePath() string
+ // Length is the length in bytes of the manifest file.
+ Length() int64
+ // PartitionID is the ID of the partition spec used to write
+ // this manifest. It must be listed in the table metadata
+ // partition-specs.
+ PartitionID() int32
+ // ManifestContent is the type of files tracked by this manifest,
+ // either data or delete files. All v1 manifests track data files.
+ ManifestContent() ManifestContent
+ // SnapshotID is the ID of the snapshot where this manifest file
+ // was added.
+ SnapshotID() int64
+ // AddedDataFiles returns the number of entries in the manifest that
+ // have the status of EntryStatusADDED.
+ AddedDataFiles() int32
+ // ExistingDataFiles returns the number of entries in the manifest
+ // which have the status of EntryStatusEXISTING.
+ ExistingDataFiles() int32
+ // DeletedDataFiles returns the number of entries in the manifest
+ // which have the status of EntryStatusDELETED.
+ DeletedDataFiles() int32
+ // AddedRows returns the number of rows in all files of the manifest
+ // that have status EntryStatusADDED.
+ AddedRows() int64
+ // ExistingRows returns the number of rows in all files of the manifest
+ // which have status EntryStatusEXISTING.
+ ExistingRows() int64
+ // DeletedRows returns the number of rows in all files of the manifest
+ // which have status EntryStatusDELETED.
+ DeletedRows() int64
+ // SequenceNum returns the sequence number when this manifest was
+ // added to the table. Will be 0 for v1 manifest lists.
+ SequenceNum() int64
+ // MinSequenceNum is the minimum data sequence number of all live data
+ // or delete files in the manifest. Will be 0 for v1 manifest lists.
+ MinSequenceNum() int64
+ // Metadata returns implementation-specific key metadata for encryption
+ // if it exists in the manifest list.
+ Metadata() []byte
+ // PartitionList returns a list of field summaries for each partition
+ // field in the spec. Each field in the list corresponds to a field in
+ // the manifest file's partition spec.
+ PartitionList() []fieldSummary
+
+ // HasAddedFiles returns true if AddedDataFiles > 0 or if it was null.
+ HasAddedFiles() bool
+ // HasExistingFiles returns true if ExistingDataFiles > 0 or if it was null.
+ HasExistingFiles() bool
+ // FetchEntries reads the manifest list file to fetch the list of
+ // manifest entries using the provided file system IO interface.
+ // If discardDeleted is true, entries for files containing deleted rows
+ // will be skipped.
+ FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error)
+}
+
+// ReadManifestList reads in an avro manifest list file and returns a slice
+// of manifest files or an error if one is encountered.
+func ReadManifestList(in io.Reader) ([]ManifestFile, error) {
+ dec, err := ocf.NewDecoder(in)
+ if err != nil {
+ return nil, err
+ }
+
+ out := make([]ManifestFile, 0)
+
+ for dec.HasNext() {
+ var file ManifestFile
+ if string(dec.Metadata()["format-version"]) == "2" {
+ file = &manifestFileV2{}
+ } else {
+ file = &manifestFileV1{}
+ }
+
+ if err := dec.Decode(file); err != nil {
+ return nil, err
+ }
+ out = append(out, file)
+ }
+
+ return out, dec.Error()
+}
+
+// ManifestEntryStatus defines constants for the entry status of
+// existing, added or deleted.
+type ManifestEntryStatus int8
+
+const (
+ EntryStatusEXISTING ManifestEntryStatus = 0
+ EntryStatusADDED ManifestEntryStatus = 1
+ EntryStatusDELETED ManifestEntryStatus = 2
+)
+
+// ManifestEntryContent defines constants for the type of file contents
+// in the file entries. Data, Position based deletes and equality based
+// deletes.
+type ManifestEntryContent int8
+
+const (
+ EntryContentData ManifestEntryContent = 0
+ EntryContentPosDeletes ManifestEntryContent = 1
+ EntryContentEqDeletes ManifestEntryContent = 2
+)
+
+// FileFormat defines constants for the format of data files.
+type FileFormat string
+
+const (
+ AvroFile FileFormat = "AVRO"
+ OrcFile FileFormat = "ORC"
+ ParquetFile FileFormat = "PARQUET"
+)
+
+type colMap[K, V any] struct {
+ Key K `avro:"key"`
+ Value V `avro:"value"`
+}
+
+func avroColMapToMap[K comparable, V any](c *[]colMap[K, V]) map[K]V {
+ if c == nil {
+ return nil
+ }
+
+ out := make(map[K]V)
+ for _, data := range *c {
+ out[data.Key] = data.Value
+ }
+ return out
+}
+
+type dataFile struct {
+ Content ManifestEntryContent `avro:"content"`
Review Comment:
Confirmed that if i start with a datafile containing some non-default value for `Content` the round trip test comes back with the default being what the struct has. so it doesn't get written and won't be read back out since the V1 schemas don't have the field.
##########
manifest.go:
##########
@@ -0,0 +1,655 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package iceberg
+
+import (
+ "io"
+ "sync"
+
+ iceio "github.com/apache/iceberg-go/io"
+
+ "github.com/hamba/avro/v2/ocf"
+)
+
+// ManifestContent indicates the type of data inside of the files
+// described by a manifest. This will indicate whether the data files
+// contain active data or deleted rows.
+type ManifestContent int32
+
+const (
+ ManifestContentData ManifestContent = 0
+ ManifestContentDeletes ManifestContent = 1
+)
+
+type fieldSummary struct {
+ ContainsNull bool `avro:"contains_null"`
+ ContainsNaN *bool `avro:"contains_nan"`
+ LowerBound *[]byte `avro:"lower_bound"`
+ UpperBound *[]byte `avro:"upper_bound"`
+}
+
+type manifestFileV1 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ AddedSnapshotID *int64 `avro:"added_snapshot_id"`
+ AddedFilesCount *int32 `avro:"added_data_files_count"`
+ ExistingFilesCount *int32 `avro:"existing_data_files_count"`
+ DeletedFilesCount *int32 `avro:"deleted_data_files_count"`
+ AddedRowsCount *int64 `avro:"added_rows_count"`
+ ExistingRowsCount *int64 `avro:"existing_rows_count"`
+ DeletedRowsCount *int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV1) Version() int { return 1 }
+func (m *manifestFileV1) FilePath() string { return m.Path }
+func (m *manifestFileV1) Length() int64 { return m.Len }
+func (m *manifestFileV1) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV1) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV1) SnapshotID() int64 {
+ if m.AddedSnapshotID == nil {
+ return 0
+ }
+ return *m.AddedSnapshotID
+}
+
+func (m *manifestFileV1) AddedDataFiles() int32 {
+ if m.AddedFilesCount == nil {
+ return 0
+ }
+ return *m.AddedFilesCount
+}
+
+func (m *manifestFileV1) ExistingDataFiles() int32 {
+ if m.ExistingFilesCount == nil {
+ return 0
+ }
+ return *m.ExistingFilesCount
+}
+
+func (m *manifestFileV1) DeletedDataFiles() int32 {
+ if m.DeletedFilesCount == nil {
+ return 0
+ }
+ return *m.DeletedFilesCount
+}
+
+func (m *manifestFileV1) AddedRows() int64 {
+ if m.AddedRowsCount == nil {
+ return 0
+ }
+ return *m.AddedRowsCount
+}
+
+func (m *manifestFileV1) ExistingRows() int64 {
+ if m.ExistingRowsCount == nil {
+ return 0
+ }
+ return *m.ExistingRowsCount
+}
+
+func (m *manifestFileV1) DeletedRows() int64 {
+ if m.DeletedRowsCount == nil {
+ return 0
+ }
+ return *m.DeletedRowsCount
+}
+
+func (m *manifestFileV1) HasAddedFiles() bool {
+ return m.AddedFilesCount == nil || *m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV1) HasExistingFiles() bool {
+ return m.ExistingFilesCount == nil || *m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV1) SequenceNum() int64 { return 0 }
+func (m *manifestFileV1) MinSequenceNum() int64 { return 0 }
+func (m *manifestFileV1) Metadata() []byte { return m.KeyMetadata }
+func (m *manifestFileV1) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV1) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+type manifestFileV2 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ SeqNumber int64 `avro:"sequence_number"`
+ MinSeqNumber int64 `avro:"min_sequence_number"`
+ AddedSnapshotID int64 `avro:"added_snapshot_id"`
+ AddedFilesCount int32 `avro:"added_files_count"`
+ ExistingFilesCount int32 `avro:"existing_files_count"`
+ DeletedFilesCount int32 `avro:"deleted_files_count"`
+ AddedRowsCount int64 `avro:"added_rows_count"`
+ ExistingRowsCount int64 `avro:"existing_rows_count"`
+ DeletedRowsCount int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV2) Version() int { return 2 }
+
+func (m *manifestFileV2) FilePath() string { return m.Path }
+func (m *manifestFileV2) Length() int64 { return m.Len }
+func (m *manifestFileV2) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV2) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV2) SnapshotID() int64 {
+ return m.AddedSnapshotID
+}
+
+func (m *manifestFileV2) AddedDataFiles() int32 {
+ return m.AddedFilesCount
+}
+
+func (m *manifestFileV2) ExistingDataFiles() int32 {
+ return m.ExistingFilesCount
+}
+
+func (m *manifestFileV2) DeletedDataFiles() int32 {
+ return m.DeletedFilesCount
+}
+
+func (m *manifestFileV2) AddedRows() int64 {
+ return m.AddedRowsCount
+}
+
+func (m *manifestFileV2) ExistingRows() int64 {
+ return m.ExistingRowsCount
+}
+
+func (m *manifestFileV2) DeletedRows() int64 {
+ return m.DeletedRowsCount
+}
+
+func (m *manifestFileV2) SequenceNum() int64 { return m.SeqNumber }
+func (m *manifestFileV2) MinSequenceNum() int64 { return m.MinSeqNumber }
+func (m *manifestFileV2) Metadata() []byte { return m.KeyMetadata }
+
+func (m *manifestFileV2) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV2) HasAddedFiles() bool {
+ return m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV2) HasExistingFiles() bool {
+ return m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV2) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+func fetchManifestEntries(m ManifestFile, fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ f, err := fs.Open(m.FilePath())
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+
+ dec, err := ocf.NewDecoder(f)
+ if err != nil {
+ return nil, err
+ }
+
+ metadata := dec.Metadata()
+ isVer1 := true
+ if string(metadata["format-version"]) == "2" {
+ isVer1 = false
+ }
+
+ results := make([]ManifestEntry, 0)
+ for dec.HasNext() {
+ var tmp ManifestEntry
+ if isVer1 {
+ tmp = &manifestEntryV1{}
+ } else {
+ tmp = &manifestEntryV2{}
+ }
+
+ if err := dec.Decode(tmp); err != nil {
+ return nil, err
+ }
+
+ if !discardDeleted || tmp.Status() != EntryStatusDELETED {
+ tmp.inheritSeqNum(m)
+ results = append(results, tmp)
+ }
+ }
+
+ return results, dec.Error()
+}
+
+// ManifestFile is the interface which covers both V1 and V2 manifest files.
+type ManifestFile interface {
+ // Version returns the version number of this manifest file.
+ // It should be 1 or 2.
+ Version() int
+ // FilePath is the location URI of this manifest file.
+ FilePath() string
+ // Length is the length in bytes of the manifest file.
+ Length() int64
+ // PartitionID is the ID of the partition spec used to write
+ // this manifest. It must be listed in the table metadata
+ // partition-specs.
+ PartitionID() int32
+ // ManifestContent is the type of files tracked by this manifest,
+ // either data or delete files. All v1 manifests track data files.
+ ManifestContent() ManifestContent
+ // SnapshotID is the ID of the snapshot where this manifest file
+ // was added.
+ SnapshotID() int64
+ // AddedDataFiles returns the number of entries in the manifest that
+ // have the status of EntryStatusADDED.
+ AddedDataFiles() int32
+ // ExistingDataFiles returns the number of entries in the manifest
+ // which have the status of EntryStatusEXISTING.
+ ExistingDataFiles() int32
+ // DeletedDataFiles returns the number of entries in the manifest
+ // which have the status of EntryStatusDELETED.
+ DeletedDataFiles() int32
+ // AddedRows returns the number of rows in all files of the manifest
+ // that have status EntryStatusADDED.
+ AddedRows() int64
+ // ExistingRows returns the number of rows in all files of the manifest
+ // which have status EntryStatusEXISTING.
+ ExistingRows() int64
+ // DeletedRows returns the number of rows in all files of the manifest
+ // which have status EntryStatusDELETED.
+ DeletedRows() int64
+ // SequenceNum returns the sequence number when this manifest was
+ // added to the table. Will be 0 for v1 manifest lists.
+ SequenceNum() int64
+ // MinSequenceNum is the minimum data sequence number of all live data
+ // or delete files in the manifest. Will be 0 for v1 manifest lists.
+ MinSequenceNum() int64
+ // Metadata returns implementation-specific key metadata for encryption
+ // if it exists in the manifest list.
+ Metadata() []byte
+ // PartitionList returns a list of field summaries for each partition
+ // field in the spec. Each field in the list corresponds to a field in
+ // the manifest file's partition spec.
+ PartitionList() []fieldSummary
+
+ // HasAddedFiles returns true if AddedDataFiles > 0 or if it was null.
+ HasAddedFiles() bool
+ // HasExistingFiles returns true if ExistingDataFiles > 0 or if it was null.
+ HasExistingFiles() bool
+ // FetchEntries reads the manifest list file to fetch the list of
+ // manifest entries using the provided file system IO interface.
+ // If discardDeleted is true, entries for files containing deleted rows
+ // will be skipped.
+ FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error)
+}
+
+// ReadManifestList reads in an avro manifest list file and returns a slice
+// of manifest files or an error if one is encountered.
+func ReadManifestList(in io.Reader) ([]ManifestFile, error) {
+ dec, err := ocf.NewDecoder(in)
+ if err != nil {
+ return nil, err
+ }
+
+ out := make([]ManifestFile, 0)
+
+ for dec.HasNext() {
+ var file ManifestFile
+ if string(dec.Metadata()["format-version"]) == "2" {
+ file = &manifestFileV2{}
+ } else {
+ file = &manifestFileV1{}
+ }
+
+ if err := dec.Decode(file); err != nil {
+ return nil, err
+ }
+ out = append(out, file)
+ }
+
+ return out, dec.Error()
+}
+
+// ManifestEntryStatus defines constants for the entry status of
+// existing, added or deleted.
+type ManifestEntryStatus int8
+
+const (
+ EntryStatusEXISTING ManifestEntryStatus = 0
+ EntryStatusADDED ManifestEntryStatus = 1
+ EntryStatusDELETED ManifestEntryStatus = 2
+)
+
+// ManifestEntryContent defines constants for the type of file contents
+// in the file entries. Data, Position based deletes and equality based
+// deletes.
+type ManifestEntryContent int8
+
+const (
+ EntryContentData ManifestEntryContent = 0
+ EntryContentPosDeletes ManifestEntryContent = 1
+ EntryContentEqDeletes ManifestEntryContent = 2
+)
+
+// FileFormat defines constants for the format of data files.
+type FileFormat string
+
+const (
+ AvroFile FileFormat = "AVRO"
+ OrcFile FileFormat = "ORC"
+ ParquetFile FileFormat = "PARQUET"
+)
+
+type colMap[K, V any] struct {
+ Key K `avro:"key"`
+ Value V `avro:"value"`
+}
+
+func avroColMapToMap[K comparable, V any](c *[]colMap[K, V]) map[K]V {
+ if c == nil {
+ return nil
+ }
+
+ out := make(map[K]V)
+ for _, data := range *c {
+ out[data.Key] = data.Value
+ }
+ return out
+}
+
+type dataFile struct {
+ Content ManifestEntryContent `avro:"content"`
Review Comment:
Confirmed that if i start with a datafile containing some non-default value for `Content` the round trip test comes back with the default being what the struct has. so it doesn't get written and won't be read back out since the V1 schemas don't have the field. updated the unit test accordingly
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] nastra commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1326936637
##########
io/s3.go:
##########
@@ -0,0 +1,108 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package io
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "net/url"
+ "os"
+ "strings"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
+ "github.com/aws/aws-sdk-go-v2/config"
+ "github.com/aws/aws-sdk-go-v2/credentials"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+ "github.com/wolfeidau/s3iofs"
+)
+
+// Constants for S3 configuration options
+const (
+ S3Region = "s3.region"
+ S3SessionToken = "s3.session-token"
+ S3SecretAccessKey = "s3.secret-access-key"
+ S3AccessKeyID = "s3.access-key-id"
+ S3EndpointURL = "s3.endpoint"
+ S3ProxyURI = "s3.proxy-uri"
+)
+
+func createS3FileIO(parsed *url.URL, props map[string]string) (IO, error) {
+ opts := []func(*config.LoadOptions) error{}
+ endpoint, ok := props[S3EndpointURL]
+ if !ok {
+ endpoint = os.Getenv("AWS_S3_ENDPOINT")
+ }
+
+ if endpoint != "" {
+ opts = append(opts, config.WithEndpointResolverWithOptions(aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
+ if service != s3.ServiceID {
+ // fallback to default resolution for the service
+ return aws.Endpoint{}, &aws.EndpointNotFoundError{}
+ }
+
+ return aws.Endpoint{
+ URL: endpoint,
+ SigningRegion: region,
+ HostnameImmutable: true,
+ }, nil
+ })))
+ }
+
+ if defaultRegion, ok := props[S3Region]; ok {
+ opts = append(opts, config.WithDefaultRegion(defaultRegion))
Review Comment:
I'm not entirely sure, but I think we might rather want to configure the region here instead of the default region using `config.WithRegion()`. Also we might want to rename `defaultRegion` to `region` to make a clear distinction that `S3Region` stands for the region (not the default region)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] zeroshade commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1328920703
##########
manifest.go:
##########
@@ -0,0 +1,655 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package iceberg
+
+import (
+ "io"
+ "sync"
+
+ iceio "github.com/apache/iceberg-go/io"
+
+ "github.com/hamba/avro/v2/ocf"
+)
+
+// ManifestContent indicates the type of data inside of the files
+// described by a manifest. This will indicate whether the data files
+// contain active data or deleted rows.
+type ManifestContent int32
+
+const (
+ ManifestContentData ManifestContent = 0
+ ManifestContentDeletes ManifestContent = 1
+)
+
+type fieldSummary struct {
+ ContainsNull bool `avro:"contains_null"`
+ ContainsNaN *bool `avro:"contains_nan"`
+ LowerBound *[]byte `avro:"lower_bound"`
+ UpperBound *[]byte `avro:"upper_bound"`
+}
+
+type manifestFileV1 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
Review Comment:
So i had a question here: according to https://iceberg.apache.org/spec/#manifests the `added_snapshot_id` is required in both V1 and V2, but pyiceberg's definition of the manifest file schema (here: https://github.com/apache/iceberg/blob/master/python/pyiceberg/manifest.py#L275) marks it as optional. Which should I follow here?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] zeroshade commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1328871174
##########
io/s3.go:
##########
@@ -0,0 +1,108 @@
+// Licensed to the Apache Software Foundation (ASF) under one
Review Comment:
Yea I was planning on doing that in a follow-up PR explicitly. I have the test stuff worked out already and, in conjunction with the iceberg REST API catalog impl I was able to replicate a similar integration test to what pyiceberg uses :smile:
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] nastra commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1326963593
##########
manifest.go:
##########
@@ -0,0 +1,655 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package iceberg
+
+import (
+ "io"
+ "sync"
+
+ iceio "github.com/apache/iceberg-go/io"
+
+ "github.com/hamba/avro/v2/ocf"
+)
+
+// ManifestContent indicates the type of data inside of the files
+// described by a manifest. This will indicate whether the data files
+// contain active data or deleted rows.
+type ManifestContent int32
+
+const (
+ ManifestContentData ManifestContent = 0
+ ManifestContentDeletes ManifestContent = 1
+)
+
+type fieldSummary struct {
+ ContainsNull bool `avro:"contains_null"`
+ ContainsNaN *bool `avro:"contains_nan"`
+ LowerBound *[]byte `avro:"lower_bound"`
+ UpperBound *[]byte `avro:"upper_bound"`
+}
+
+type manifestFileV1 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
Review Comment:
`content` isn't part of a v1 manifest file. See also https://github.com/apache/iceberg/blob/9907a97351138e676d08e02ea53b746f4e331ec6/core/src/main/java/org/apache/iceberg/V1Metadata.java#L33
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] zeroshade commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1331917468
##########
manifest.go:
##########
@@ -0,0 +1,655 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package iceberg
+
+import (
+ "io"
+ "sync"
+
+ iceio "github.com/apache/iceberg-go/io"
+
+ "github.com/hamba/avro/v2/ocf"
+)
+
+// ManifestContent indicates the type of data inside of the files
+// described by a manifest. This will indicate whether the data files
+// contain active data or deleted rows.
+type ManifestContent int32
+
+const (
+ ManifestContentData ManifestContent = 0
+ ManifestContentDeletes ManifestContent = 1
+)
+
+type fieldSummary struct {
+ ContainsNull bool `avro:"contains_null"`
+ ContainsNaN *bool `avro:"contains_nan"`
+ LowerBound *[]byte `avro:"lower_bound"`
+ UpperBound *[]byte `avro:"upper_bound"`
+}
+
+type manifestFileV1 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ AddedSnapshotID *int64 `avro:"added_snapshot_id"`
+ AddedFilesCount *int32 `avro:"added_data_files_count"`
+ ExistingFilesCount *int32 `avro:"existing_data_files_count"`
+ DeletedFilesCount *int32 `avro:"deleted_data_files_count"`
+ AddedRowsCount *int64 `avro:"added_rows_count"`
+ ExistingRowsCount *int64 `avro:"existing_rows_count"`
+ DeletedRowsCount *int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV1) Version() int { return 1 }
+func (m *manifestFileV1) FilePath() string { return m.Path }
+func (m *manifestFileV1) Length() int64 { return m.Len }
+func (m *manifestFileV1) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV1) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV1) SnapshotID() int64 {
+ if m.AddedSnapshotID == nil {
+ return 0
+ }
+ return *m.AddedSnapshotID
+}
+
+func (m *manifestFileV1) AddedDataFiles() int32 {
+ if m.AddedFilesCount == nil {
+ return 0
+ }
+ return *m.AddedFilesCount
+}
+
+func (m *manifestFileV1) ExistingDataFiles() int32 {
+ if m.ExistingFilesCount == nil {
+ return 0
+ }
+ return *m.ExistingFilesCount
+}
+
+func (m *manifestFileV1) DeletedDataFiles() int32 {
+ if m.DeletedFilesCount == nil {
+ return 0
+ }
+ return *m.DeletedFilesCount
+}
+
+func (m *manifestFileV1) AddedRows() int64 {
+ if m.AddedRowsCount == nil {
+ return 0
+ }
+ return *m.AddedRowsCount
+}
+
+func (m *manifestFileV1) ExistingRows() int64 {
+ if m.ExistingRowsCount == nil {
+ return 0
+ }
+ return *m.ExistingRowsCount
+}
+
+func (m *manifestFileV1) DeletedRows() int64 {
+ if m.DeletedRowsCount == nil {
+ return 0
+ }
+ return *m.DeletedRowsCount
+}
+
+func (m *manifestFileV1) HasAddedFiles() bool {
+ return m.AddedFilesCount == nil || *m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV1) HasExistingFiles() bool {
+ return m.ExistingFilesCount == nil || *m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV1) SequenceNum() int64 { return 0 }
+func (m *manifestFileV1) MinSequenceNum() int64 { return 0 }
+func (m *manifestFileV1) Metadata() []byte { return m.KeyMetadata }
+func (m *manifestFileV1) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV1) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+type manifestFileV2 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ SeqNumber int64 `avro:"sequence_number"`
+ MinSeqNumber int64 `avro:"min_sequence_number"`
+ AddedSnapshotID int64 `avro:"added_snapshot_id"`
+ AddedFilesCount int32 `avro:"added_files_count"`
+ ExistingFilesCount int32 `avro:"existing_files_count"`
+ DeletedFilesCount int32 `avro:"deleted_files_count"`
+ AddedRowsCount int64 `avro:"added_rows_count"`
+ ExistingRowsCount int64 `avro:"existing_rows_count"`
+ DeletedRowsCount int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV2) Version() int { return 2 }
+
+func (m *manifestFileV2) FilePath() string { return m.Path }
+func (m *manifestFileV2) Length() int64 { return m.Len }
+func (m *manifestFileV2) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV2) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV2) SnapshotID() int64 {
+ return m.AddedSnapshotID
+}
+
+func (m *manifestFileV2) AddedDataFiles() int32 {
+ return m.AddedFilesCount
+}
+
+func (m *manifestFileV2) ExistingDataFiles() int32 {
+ return m.ExistingFilesCount
+}
+
+func (m *manifestFileV2) DeletedDataFiles() int32 {
+ return m.DeletedFilesCount
+}
+
+func (m *manifestFileV2) AddedRows() int64 {
+ return m.AddedRowsCount
+}
+
+func (m *manifestFileV2) ExistingRows() int64 {
+ return m.ExistingRowsCount
+}
+
+func (m *manifestFileV2) DeletedRows() int64 {
+ return m.DeletedRowsCount
+}
+
+func (m *manifestFileV2) SequenceNum() int64 { return m.SeqNumber }
+func (m *manifestFileV2) MinSequenceNum() int64 { return m.MinSeqNumber }
+func (m *manifestFileV2) Metadata() []byte { return m.KeyMetadata }
+
+func (m *manifestFileV2) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV2) HasAddedFiles() bool {
+ return m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV2) HasExistingFiles() bool {
+ return m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV2) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+func fetchManifestEntries(m ManifestFile, fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ f, err := fs.Open(m.FilePath())
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+
+ dec, err := ocf.NewDecoder(f)
+ if err != nil {
+ return nil, err
+ }
+
+ metadata := dec.Metadata()
+ isVer1 := true
+ if string(metadata["format-version"]) == "2" {
+ isVer1 = false
+ }
+
+ results := make([]ManifestEntry, 0)
+ for dec.HasNext() {
+ var tmp ManifestEntry
+ if isVer1 {
+ tmp = &manifestEntryV1{}
+ } else {
+ tmp = &manifestEntryV2{}
+ }
+
+ if err := dec.Decode(tmp); err != nil {
+ return nil, err
+ }
+
+ if !discardDeleted || tmp.Status() != EntryStatusDELETED {
+ tmp.inheritSeqNum(m)
+ results = append(results, tmp)
+ }
+ }
+
+ return results, dec.Error()
+}
+
+// ManifestFile is the interface which covers both V1 and V2 manifest files.
+type ManifestFile interface {
+ // Version returns the version number of this manifest file.
+ // It should be 1 or 2.
+ Version() int
+ // FilePath is the location URI of this manifest file.
+ FilePath() string
+ // Length is the length in bytes of the manifest file.
+ Length() int64
+ // PartitionID is the ID of the partition spec used to write
+ // this manifest. It must be listed in the table metadata
+ // partition-specs.
+ PartitionID() int32
+ // ManifestContent is the type of files tracked by this manifest,
+ // either data or delete files. All v1 manifests track data files.
+ ManifestContent() ManifestContent
+ // SnapshotID is the ID of the snapshot where this manifest file
+ // was added.
+ SnapshotID() int64
+ // AddedDataFiles returns the number of entries in the manifest that
+ // have the status of EntryStatusADDED.
+ AddedDataFiles() int32
+ // ExistingDataFiles returns the number of entries in the manifest
+ // which have the status of EntryStatusEXISTING.
+ ExistingDataFiles() int32
+ // DeletedDataFiles returns the number of entries in the manifest
+ // which have the status of EntryStatusDELETED.
+ DeletedDataFiles() int32
+ // AddedRows returns the number of rows in all files of the manifest
+ // that have status EntryStatusADDED.
+ AddedRows() int64
+ // ExistingRows returns the number of rows in all files of the manifest
+ // which have status EntryStatusEXISTING.
+ ExistingRows() int64
+ // DeletedRows returns the number of rows in all files of the manifest
+ // which have status EntryStatusDELETED.
+ DeletedRows() int64
+ // SequenceNum returns the sequence number when this manifest was
+ // added to the table. Will be 0 for v1 manifest lists.
+ SequenceNum() int64
+ // MinSequenceNum is the minimum data sequence number of all live data
+ // or delete files in the manifest. Will be 0 for v1 manifest lists.
+ MinSequenceNum() int64
+ // Metadata returns implementation-specific key metadata for encryption
+ // if it exists in the manifest list.
+ Metadata() []byte
+ // PartitionList returns a list of field summaries for each partition
+ // field in the spec. Each field in the list corresponds to a field in
+ // the manifest file's partition spec.
+ PartitionList() []fieldSummary
+
+ // HasAddedFiles returns true if AddedDataFiles > 0 or if it was null.
+ HasAddedFiles() bool
+ // HasExistingFiles returns true if ExistingDataFiles > 0 or if it was null.
+ HasExistingFiles() bool
+ // FetchEntries reads the manifest list file to fetch the list of
+ // manifest entries using the provided file system IO interface.
+ // If discardDeleted is true, entries for files containing deleted rows
+ // will be skipped.
+ FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error)
+}
+
+// ReadManifestList reads in an avro manifest list file and returns a slice
+// of manifest files or an error if one is encountered.
+func ReadManifestList(in io.Reader) ([]ManifestFile, error) {
+ dec, err := ocf.NewDecoder(in)
+ if err != nil {
+ return nil, err
+ }
+
+ out := make([]ManifestFile, 0)
+
+ for dec.HasNext() {
+ var file ManifestFile
+ if string(dec.Metadata()["format-version"]) == "2" {
+ file = &manifestFileV2{}
+ } else {
+ file = &manifestFileV1{}
+ }
+
+ if err := dec.Decode(file); err != nil {
+ return nil, err
+ }
+ out = append(out, file)
+ }
+
+ return out, dec.Error()
+}
+
+// ManifestEntryStatus defines constants for the entry status of
+// existing, added or deleted.
+type ManifestEntryStatus int8
+
+const (
+ EntryStatusEXISTING ManifestEntryStatus = 0
+ EntryStatusADDED ManifestEntryStatus = 1
+ EntryStatusDELETED ManifestEntryStatus = 2
+)
+
+// ManifestEntryContent defines constants for the type of file contents
+// in the file entries. Data, Position based deletes and equality based
+// deletes.
+type ManifestEntryContent int8
+
+const (
+ EntryContentData ManifestEntryContent = 0
+ EntryContentPosDeletes ManifestEntryContent = 1
+ EntryContentEqDeletes ManifestEntryContent = 2
+)
+
+// FileFormat defines constants for the format of data files.
+type FileFormat string
+
+const (
+ AvroFile FileFormat = "AVRO"
+ OrcFile FileFormat = "ORC"
+ ParquetFile FileFormat = "PARQUET"
+)
+
+type colMap[K, V any] struct {
+ Key K `avro:"key"`
+ Value V `avro:"value"`
+}
+
+func avroColMapToMap[K comparable, V any](c *[]colMap[K, V]) map[K]V {
+ if c == nil {
+ return nil
+ }
+
+ out := make(map[K]V)
+ for _, data := range *c {
+ out[data.Key] = data.Value
+ }
+ return out
+}
+
+type dataFile struct {
+ Content ManifestEntryContent `avro:"content"`
Review Comment:
In theory, it potentially would avoid writing this field for V1 since it isn't in the schema, but I'll double check
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] zeroshade commented on pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade commented on PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#issuecomment-1687021937
CC @coded9 @Fokko @nastra @bitsondatadev @rdblue
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] nastra commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1331896695
##########
manifest.go:
##########
@@ -0,0 +1,655 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package iceberg
+
+import (
+ "io"
+ "sync"
+
+ iceio "github.com/apache/iceberg-go/io"
+
+ "github.com/hamba/avro/v2/ocf"
+)
+
+// ManifestContent indicates the type of data inside of the files
+// described by a manifest. This will indicate whether the data files
+// contain active data or deleted rows.
+type ManifestContent int32
+
+const (
+ ManifestContentData ManifestContent = 0
+ ManifestContentDeletes ManifestContent = 1
+)
+
+type fieldSummary struct {
+ ContainsNull bool `avro:"contains_null"`
+ ContainsNaN *bool `avro:"contains_nan"`
+ LowerBound *[]byte `avro:"lower_bound"`
+ UpperBound *[]byte `avro:"upper_bound"`
+}
+
+type manifestFileV1 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
Review Comment:
it seems to be optional in the Java code as well: https://github.com/apache/iceberg/blob/5e5c6d1849dece373b0d425ebd3ba5c7e98ad0ef/api/src/main/java/org/apache/iceberg/ManifestFile.java#L52C1-L53
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] zeroshade commented on pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade commented on PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#issuecomment-1719772450
Rebased with the merge for the partitioning stuff, so should be all good here
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] nastra merged pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra merged PR #3:
URL: https://github.com/apache/iceberg-go/pull/3
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] nastra commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1327001267
##########
manifest.go:
##########
@@ -0,0 +1,655 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package iceberg
+
+import (
+ "io"
+ "sync"
+
+ iceio "github.com/apache/iceberg-go/io"
+
+ "github.com/hamba/avro/v2/ocf"
+)
+
+// ManifestContent indicates the type of data inside of the files
+// described by a manifest. This will indicate whether the data files
+// contain active data or deleted rows.
+type ManifestContent int32
+
+const (
+ ManifestContentData ManifestContent = 0
+ ManifestContentDeletes ManifestContent = 1
+)
+
+type fieldSummary struct {
+ ContainsNull bool `avro:"contains_null"`
+ ContainsNaN *bool `avro:"contains_nan"`
+ LowerBound *[]byte `avro:"lower_bound"`
+ UpperBound *[]byte `avro:"upper_bound"`
+}
+
+type manifestFileV1 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ AddedSnapshotID *int64 `avro:"added_snapshot_id"`
+ AddedFilesCount *int32 `avro:"added_data_files_count"`
+ ExistingFilesCount *int32 `avro:"existing_data_files_count"`
+ DeletedFilesCount *int32 `avro:"deleted_data_files_count"`
+ AddedRowsCount *int64 `avro:"added_rows_count"`
+ ExistingRowsCount *int64 `avro:"existing_rows_count"`
+ DeletedRowsCount *int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV1) Version() int { return 1 }
+func (m *manifestFileV1) FilePath() string { return m.Path }
+func (m *manifestFileV1) Length() int64 { return m.Len }
+func (m *manifestFileV1) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV1) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV1) SnapshotID() int64 {
+ if m.AddedSnapshotID == nil {
+ return 0
+ }
+ return *m.AddedSnapshotID
+}
+
+func (m *manifestFileV1) AddedDataFiles() int32 {
+ if m.AddedFilesCount == nil {
+ return 0
+ }
+ return *m.AddedFilesCount
+}
+
+func (m *manifestFileV1) ExistingDataFiles() int32 {
+ if m.ExistingFilesCount == nil {
+ return 0
+ }
+ return *m.ExistingFilesCount
+}
+
+func (m *manifestFileV1) DeletedDataFiles() int32 {
+ if m.DeletedFilesCount == nil {
+ return 0
+ }
+ return *m.DeletedFilesCount
+}
+
+func (m *manifestFileV1) AddedRows() int64 {
+ if m.AddedRowsCount == nil {
+ return 0
+ }
+ return *m.AddedRowsCount
+}
+
+func (m *manifestFileV1) ExistingRows() int64 {
+ if m.ExistingRowsCount == nil {
+ return 0
+ }
+ return *m.ExistingRowsCount
+}
+
+func (m *manifestFileV1) DeletedRows() int64 {
+ if m.DeletedRowsCount == nil {
+ return 0
+ }
+ return *m.DeletedRowsCount
+}
+
+func (m *manifestFileV1) HasAddedFiles() bool {
+ return m.AddedFilesCount == nil || *m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV1) HasExistingFiles() bool {
+ return m.ExistingFilesCount == nil || *m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV1) SequenceNum() int64 { return 0 }
+func (m *manifestFileV1) MinSequenceNum() int64 { return 0 }
+func (m *manifestFileV1) Metadata() []byte { return m.KeyMetadata }
+func (m *manifestFileV1) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV1) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+type manifestFileV2 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ SeqNumber int64 `avro:"sequence_number"`
+ MinSeqNumber int64 `avro:"min_sequence_number"`
+ AddedSnapshotID int64 `avro:"added_snapshot_id"`
+ AddedFilesCount int32 `avro:"added_files_count"`
+ ExistingFilesCount int32 `avro:"existing_files_count"`
+ DeletedFilesCount int32 `avro:"deleted_files_count"`
+ AddedRowsCount int64 `avro:"added_rows_count"`
+ ExistingRowsCount int64 `avro:"existing_rows_count"`
+ DeletedRowsCount int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV2) Version() int { return 2 }
+
+func (m *manifestFileV2) FilePath() string { return m.Path }
+func (m *manifestFileV2) Length() int64 { return m.Len }
+func (m *manifestFileV2) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV2) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV2) SnapshotID() int64 {
+ return m.AddedSnapshotID
+}
+
+func (m *manifestFileV2) AddedDataFiles() int32 {
+ return m.AddedFilesCount
+}
+
+func (m *manifestFileV2) ExistingDataFiles() int32 {
+ return m.ExistingFilesCount
+}
+
+func (m *manifestFileV2) DeletedDataFiles() int32 {
+ return m.DeletedFilesCount
+}
+
+func (m *manifestFileV2) AddedRows() int64 {
+ return m.AddedRowsCount
+}
+
+func (m *manifestFileV2) ExistingRows() int64 {
+ return m.ExistingRowsCount
+}
+
+func (m *manifestFileV2) DeletedRows() int64 {
+ return m.DeletedRowsCount
+}
+
+func (m *manifestFileV2) SequenceNum() int64 { return m.SeqNumber }
+func (m *manifestFileV2) MinSequenceNum() int64 { return m.MinSeqNumber }
+func (m *manifestFileV2) Metadata() []byte { return m.KeyMetadata }
+
+func (m *manifestFileV2) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV2) HasAddedFiles() bool {
+ return m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV2) HasExistingFiles() bool {
+ return m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV2) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+func fetchManifestEntries(m ManifestFile, fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ f, err := fs.Open(m.FilePath())
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+
+ dec, err := ocf.NewDecoder(f)
+ if err != nil {
+ return nil, err
+ }
+
+ metadata := dec.Metadata()
+ isVer1 := true
+ if string(metadata["format-version"]) == "2" {
+ isVer1 = false
+ }
+
+ results := make([]ManifestEntry, 0)
+ for dec.HasNext() {
+ var tmp ManifestEntry
+ if isVer1 {
+ tmp = &manifestEntryV1{}
+ } else {
+ tmp = &manifestEntryV2{}
+ }
+
+ if err := dec.Decode(tmp); err != nil {
+ return nil, err
+ }
+
+ if !discardDeleted || tmp.Status() != EntryStatusDELETED {
+ tmp.inheritSeqNum(m)
+ results = append(results, tmp)
+ }
+ }
+
+ return results, dec.Error()
+}
+
+// ManifestFile is the interface which covers both V1 and V2 manifest files.
+type ManifestFile interface {
+ // Version returns the version number of this manifest file.
+ // It should be 1 or 2.
+ Version() int
+ // FilePath is the location URI of this manifest file.
+ FilePath() string
+ // Length is the length in bytes of the manifest file.
+ Length() int64
+ // PartitionID is the ID of the partition spec used to write
+ // this manifest. It must be listed in the table metadata
+ // partition-specs.
+ PartitionID() int32
+ // ManifestContent is the type of files tracked by this manifest,
+ // either data or delete files. All v1 manifests track data files.
+ ManifestContent() ManifestContent
+ // SnapshotID is the ID of the snapshot where this manifest file
+ // was added.
+ SnapshotID() int64
+ // AddedDataFiles returns the number of entries in the manifest that
+ // have the status of EntryStatusADDED.
+ AddedDataFiles() int32
+ // ExistingDataFiles returns the number of entries in the manifest
+ // which have the status of EntryStatusEXISTING.
+ ExistingDataFiles() int32
+ // DeletedDataFiles returns the number of entries in the manifest
+ // which have the status of EntryStatusDELETED.
+ DeletedDataFiles() int32
+ // AddedRows returns the number of rows in all files of the manifest
+ // that have status EntryStatusADDED.
+ AddedRows() int64
+ // ExistingRows returns the number of rows in all files of the manifest
+ // which have status EntryStatusEXISTING.
+ ExistingRows() int64
+ // DeletedRows returns the number of rows in all files of the manifest
+ // which have status EntryStatusDELETED.
+ DeletedRows() int64
+ // SequenceNum returns the sequence number when this manifest was
+ // added to the table. Will be 0 for v1 manifest lists.
+ SequenceNum() int64
+ // MinSequenceNum is the minimum data sequence number of all live data
+ // or delete files in the manifest. Will be 0 for v1 manifest lists.
+ MinSequenceNum() int64
+ // Metadata returns implementation-specific key metadata for encryption
+ // if it exists in the manifest list.
+ Metadata() []byte
+ // PartitionList returns a list of field summaries for each partition
+ // field in the spec. Each field in the list corresponds to a field in
+ // the manifest file's partition spec.
+ PartitionList() []fieldSummary
+
+ // HasAddedFiles returns true if AddedDataFiles > 0 or if it was null.
+ HasAddedFiles() bool
+ // HasExistingFiles returns true if ExistingDataFiles > 0 or if it was null.
+ HasExistingFiles() bool
+ // FetchEntries reads the manifest list file to fetch the list of
+ // manifest entries using the provided file system IO interface.
+ // If discardDeleted is true, entries for files containing deleted rows
+ // will be skipped.
+ FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error)
+}
+
+// ReadManifestList reads in an avro manifest list file and returns a slice
+// of manifest files or an error if one is encountered.
+func ReadManifestList(in io.Reader) ([]ManifestFile, error) {
+ dec, err := ocf.NewDecoder(in)
+ if err != nil {
+ return nil, err
+ }
+
+ out := make([]ManifestFile, 0)
+
+ for dec.HasNext() {
+ var file ManifestFile
+ if string(dec.Metadata()["format-version"]) == "2" {
+ file = &manifestFileV2{}
+ } else {
+ file = &manifestFileV1{}
+ }
+
+ if err := dec.Decode(file); err != nil {
+ return nil, err
+ }
+ out = append(out, file)
+ }
+
+ return out, dec.Error()
+}
+
+// ManifestEntryStatus defines constants for the entry status of
+// existing, added or deleted.
+type ManifestEntryStatus int8
+
+const (
+ EntryStatusEXISTING ManifestEntryStatus = 0
+ EntryStatusADDED ManifestEntryStatus = 1
+ EntryStatusDELETED ManifestEntryStatus = 2
+)
+
+// ManifestEntryContent defines constants for the type of file contents
+// in the file entries. Data, Position based deletes and equality based
+// deletes.
+type ManifestEntryContent int8
+
+const (
+ EntryContentData ManifestEntryContent = 0
+ EntryContentPosDeletes ManifestEntryContent = 1
+ EntryContentEqDeletes ManifestEntryContent = 2
+)
+
+// FileFormat defines constants for the format of data files.
+type FileFormat string
+
+const (
+ AvroFile FileFormat = "AVRO"
+ OrcFile FileFormat = "ORC"
+ ParquetFile FileFormat = "PARQUET"
+)
+
+type colMap[K, V any] struct {
+ Key K `avro:"key"`
+ Value V `avro:"value"`
+}
+
+func avroColMapToMap[K comparable, V any](c *[]colMap[K, V]) map[K]V {
+ if c == nil {
+ return nil
+ }
+
+ out := make(map[K]V)
+ for _, data := range *c {
+ out[data.Key] = data.Value
+ }
+ return out
+}
+
+type dataFile struct {
+ Content ManifestEntryContent `avro:"content"`
+ Path string `avro:"file_path"`
+ Format FileFormat `avro:"file_format"`
+ PartitionData map[string]any `avro:"partition"`
+ RecordCount int64 `avro:"record_count"`
+ FileSize int64 `avro:"file_size_in_bytes"`
+ BlockSizeInBytes int64 `avro:"block_size_in_bytes"`
+ ColSizes *[]colMap[int, int64] `avro:"column_sizes"`
+ ValCounts *[]colMap[int, int64] `avro:"value_counts"`
+ NullCounts *[]colMap[int, int64] `avro:"null_value_counts"`
+ NaNCounts *[]colMap[int, int64] `avro:"nan_value_counts"`
+ DistinctCounts *[]colMap[int, int64] `avro:"distinct_counts"`
+ LowerBounds *[]colMap[int, []byte] `avro:"lower_bounds"`
+ UpperBounds *[]colMap[int, []byte] `avro:"upper_bounds"`
+ KeyMetadata *[]byte `avro:"key_metadata"`
+ SplitOffsets *[]int64 `avro:"split_offsets"`
+ EqualityIDs *[]int `avro:"equality_ids"`
+ SortOrder *int `avro:"sort_order_id"`
+
+ colSizeMap map[int]int64
+ valCntMap map[int]int64
+ nullCntMap map[int]int64
+ nanCntMap map[int]int64
+ distinctCntMap map[int]int64
+ lowerBoundMap map[int][]byte
+ upperBoundMap map[int][]byte
+
+ initMaps sync.Once
+}
+
+func (d *dataFile) initializeMapData() {
+ d.initMaps.Do(func() {
+ d.colSizeMap = avroColMapToMap(d.ColSizes)
+ d.valCntMap = avroColMapToMap(d.ValCounts)
+ d.nullCntMap = avroColMapToMap(d.NullCounts)
+ d.nanCntMap = avroColMapToMap(d.NaNCounts)
+ d.distinctCntMap = avroColMapToMap(d.DistinctCounts)
+ d.lowerBoundMap = avroColMapToMap(d.LowerBounds)
+ d.upperBoundMap = avroColMapToMap(d.UpperBounds)
+ })
+}
+
+func (d *dataFile) ContentType() ManifestEntryContent { return d.Content }
+func (d *dataFile) FilePath() string { return d.Path }
+func (d *dataFile) FileFormat() FileFormat { return d.Format }
+func (d *dataFile) Partition() map[string]any { return d.PartitionData }
+func (d *dataFile) Count() int64 { return d.RecordCount }
+func (d *dataFile) FileSizeBytes() int64 { return d.FileSize }
+
+func (d *dataFile) ColumnSizes() map[int]int64 {
+ d.initializeMapData()
+ return d.colSizeMap
+}
+
+func (d *dataFile) ValueCounts() map[int]int64 {
+ d.initializeMapData()
+ return d.valCntMap
+}
+
+func (d *dataFile) NullValueCounts() map[int]int64 {
+ d.initializeMapData()
+ return d.nullCntMap
+}
+
+func (d *dataFile) NaNValueCounts() map[int]int64 {
+ d.initializeMapData()
+ return d.nanCntMap
+}
+
+func (d *dataFile) DistinctValueCounts() map[int]int64 {
+ d.initializeMapData()
+ return d.distinctCntMap
+}
+
+func (d *dataFile) LowerBoundValues() map[int][]byte {
+ d.initializeMapData()
+ return d.lowerBoundMap
+}
+
+func (d *dataFile) UpperBoundValues() map[int][]byte {
+ d.initializeMapData()
+ return d.upperBoundMap
+}
+
+func (d *dataFile) MetadataKey() []byte {
+ if d.KeyMetadata == nil {
+ return nil
+ }
+ return *d.KeyMetadata
+}
+
+func (d *dataFile) Splits() []int64 {
+ if d.SplitOffsets == nil {
+ return nil
+ }
+ return *d.SplitOffsets
+}
+
+func (d *dataFile) EqualityFieldIDs() []int {
+ if d.EqualityIDs == nil {
+ return nil
+ }
+ return d.EqualityFieldIDs()
+}
+
+func (d *dataFile) SortOrderID() *int { return d.SortOrder }
+
+type manifestEntryV1 struct {
+ EntryStatus ManifestEntryStatus `avro:"status"`
+ Snapshot int64 `avro:"snapshot_id"`
+ SeqNum *int64
+ FileSeqNum *int64
+ Data dataFile `avro:"data_file"`
+}
+
+func (m *manifestEntryV1) inheritSeqNum(manifest ManifestFile) {}
+
+func (m *manifestEntryV1) Status() ManifestEntryStatus { return m.EntryStatus }
+func (m *manifestEntryV1) SnapshotID() int64 { return m.Snapshot }
+
+func (m *manifestEntryV1) SequenceNum() int64 {
+ if m.SeqNum == nil {
+ return 0
+ }
+ return *m.SeqNum
+}
+
+func (m *manifestEntryV1) FileSequenceNum() *int64 {
+ return m.FileSeqNum
+}
+
+func (m *manifestEntryV1) DataFile() DataFile { return &m.Data }
+
+type manifestEntryV2 struct {
+ EntryStatus ManifestEntryStatus `avro:"status"`
+ Snapshot *int64 `avro:"snapshot_id"`
+ SeqNum *int64 `avro:"sequence_number"`
+ FileSeqNum *int64 `avro:"file_sequence_number"`
+ Data dataFile `avro:"data_file"`
+}
+
+func (m *manifestEntryV2) inheritSeqNum(manifest ManifestFile) {
+ if m.Snapshot == nil {
+ snap := manifest.SnapshotID()
+ m.Snapshot = &snap
+ }
+
+ manifestSequenceNum := manifest.SequenceNum()
+ if m.SeqNum == nil && (manifestSequenceNum == 0 || m.EntryStatus == EntryStatusADDED) {
+ m.SeqNum = &manifestSequenceNum
+ }
+
+ if m.FileSeqNum == nil && (manifestSequenceNum == 0 || m.EntryStatus == EntryStatusADDED) {
+ m.FileSeqNum = &manifestSequenceNum
+ }
+}
+
+func (m *manifestEntryV2) Status() ManifestEntryStatus { return m.EntryStatus }
+func (m *manifestEntryV2) SnapshotID() int64 {
+ if m.Snapshot == nil {
+ return 0
+ }
+ return *m.Snapshot
+}
+
+func (m *manifestEntryV2) SequenceNum() int64 {
+ if m.SeqNum == nil {
+ return 0
+ }
+ return *m.SeqNum
+}
+
+func (m *manifestEntryV2) FileSequenceNum() *int64 {
+ return m.FileSeqNum
+}
+
+func (m *manifestEntryV2) DataFile() DataFile { return &m.Data }
+
+// DataFile is the interface for reading the information about a
+// given data file indicated by an entry in a manifest list.
+type DataFile interface {
+ // ContentType is the type of the content stored by the data file,
+ // either Data, Equality deletes, or Position deletes. All v1 files
+ // are Data files.
+ ContentType() ManifestEntryContent
+ // FilePath is the full URI for the file, complete with FS scheme.
+ FilePath() string
+ // FileFormat is the format of the data file, AVRO, Orc, or Parquet.
+ FileFormat() FileFormat
+ // Partition returns a mapping of field name to partition value for
+ // each of the partition spec's fields.
+ Partition() map[string]any
+ // Count returns the number of records in this file.
+ Count() int64
+ // FileSizeBytes is the total file size in bytes.
+ FileSizeBytes() int64
+ // ColumnSizes is a mapping from column id to the total size on disk
+ // of all regions that store the column. Does not include bytes
+ // necessary to read other columns, like footers. Map will be nil for
+ // row-oriented formats (avro).
+ ColumnSizes() map[int]int64
+ // ValueCounts is a mapping from column id to the number of values
+ // in the column, including null and NaN values.
+ ValueCounts() map[int]int64
+ // NullValueCounts is a mapping from column id to the number of
+ // null values in the column.
+ NullValueCounts() map[int]int64
+ // NaNValueCounts is a mapping from column id to the number of NaN
+ // values in the column.
+ NaNValueCounts() map[int]int64
+ // DistictValueCounts is a mapping from column id to the number of
+ // distinct values in the column. Distinct counts must be derived
+ // using values in the file by counting or using sketches, but not
+ // using methods like merging existing distinct counts.
+ DistinctValueCounts() map[int]int64
+ // LowerBoundValues is a mapping from column id to the lower bounded
+ // value of the column, serialized as binary. Each value in the column
+ // must be less than or requal to all non-null, non-NaN values in the
+ // column for the file.
+ LowerBoundValues() map[int][]byte
+ // UpperBoundValues is a mapping from column id to the upper bounded
+ // value of the column, serialized as binary. Each value in the column
+ // must be greater than or equal to all non-null, non-NaN values in
+ // the column for the file.
+ UpperBoundValues() map[int][]byte
+ // MetadataKey is implementation-specific key metadata for encryption.
+ MetadataKey() []byte
+ // Splits are the split offsets for the data file. For example,
+ // all row group offsets in a Parquet file. Must be sorted ascending.
+ Splits() []int64
Review Comment:
I'd rather name this `SplitOffsets()`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] nastra commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1325904401
##########
io/io.go:
##########
@@ -0,0 +1,327 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package io
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "io/fs"
+ "net/http"
+ "net/url"
+ "os"
+ "strings"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
+ "github.com/aws/aws-sdk-go-v2/config"
+ "github.com/aws/aws-sdk-go-v2/credentials"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+ "github.com/wolfeidau/s3iofs"
+)
+
+const (
+ S3RegionName = "s3.region-name"
Review Comment:
```suggestion
S3RegionName = "s3.region"
```
to be consistent with pyiceberg
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] nastra commented on pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#issuecomment-1720955413
@zeroshade I did a first pass around the manifest schema and code and left a few comments. I still need to review the APIs for IO and the tests
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] zeroshade commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1328874840
##########
io/s3.go:
##########
@@ -0,0 +1,108 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package io
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "net/url"
+ "os"
+ "strings"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
+ "github.com/aws/aws-sdk-go-v2/config"
+ "github.com/aws/aws-sdk-go-v2/credentials"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+ "github.com/wolfeidau/s3iofs"
+)
+
+// Constants for S3 configuration options
+const (
+ S3Region = "s3.region"
+ S3SessionToken = "s3.session-token"
+ S3SecretAccessKey = "s3.secret-access-key"
+ S3AccessKeyID = "s3.access-key-id"
+ S3EndpointURL = "s3.endpoint"
+ S3ProxyURI = "s3.proxy-uri"
+)
+
+func createS3FileIO(parsed *url.URL, props map[string]string) (IO, error) {
+ opts := []func(*config.LoadOptions) error{}
+ endpoint, ok := props[S3EndpointURL]
+ if !ok {
+ endpoint = os.Getenv("AWS_S3_ENDPOINT")
+ }
+
+ if endpoint != "" {
+ opts = append(opts, config.WithEndpointResolverWithOptions(aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
+ if service != s3.ServiceID {
+ // fallback to default resolution for the service
+ return aws.Endpoint{}, &aws.EndpointNotFoundError{}
+ }
+
+ return aws.Endpoint{
+ URL: endpoint,
+ SigningRegion: region,
+ HostnameImmutable: true,
+ }, nil
+ })))
+ }
+
+ if defaultRegion, ok := props[S3Region]; ok {
+ opts = append(opts, config.WithDefaultRegion(defaultRegion))
Review Comment:
so in this situation since we're not really exposing the ability for consumers to set their own options explicitly, I think it ends up being basically identical if we use `WithDefaultRegion` vs `WithRegion` as the only *real* difference between them is that any `WithDefaultRegion` will always be superceded by any `WithRegion` options in the config.
By the same token since it wouldn't have any real difference, I agree that it makes sense from a code perspective for me to switch these to use `WithRegion` rather than `WithDefaultRegion`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] nastra commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1326972514
##########
internal/avro_schemas.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package internal
+
+import "github.com/hamba/avro/v2"
+
+const (
+ ManifestListV1Key = "manifest-list-v1"
+ ManifestListV2Key = "manifest-list-v2"
+ ManifestEntryV1Key = "manifest-entry-v1"
+ ManifestEntryV2Key = "manifest-entry-v2"
+)
+
+var (
+ AvroSchemaCache avro.SchemaCache
+)
+
+func init() {
+ AvroSchemaCache.Add(ManifestListV1Key, avro.MustParse(`{
Review Comment:
this looks correct
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] nastra commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1331871844
##########
manifest.go:
##########
@@ -0,0 +1,655 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package iceberg
+
+import (
+ "io"
+ "sync"
+
+ iceio "github.com/apache/iceberg-go/io"
+
+ "github.com/hamba/avro/v2/ocf"
+)
+
+// ManifestContent indicates the type of data inside of the files
+// described by a manifest. This will indicate whether the data files
+// contain active data or deleted rows.
+type ManifestContent int32
+
+const (
+ ManifestContentData ManifestContent = 0
+ ManifestContentDeletes ManifestContent = 1
+)
+
+type fieldSummary struct {
+ ContainsNull bool `avro:"contains_null"`
+ ContainsNaN *bool `avro:"contains_nan"`
+ LowerBound *[]byte `avro:"lower_bound"`
+ UpperBound *[]byte `avro:"upper_bound"`
+}
+
+type manifestFileV1 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ AddedSnapshotID *int64 `avro:"added_snapshot_id"`
+ AddedFilesCount *int32 `avro:"added_data_files_count"`
+ ExistingFilesCount *int32 `avro:"existing_data_files_count"`
+ DeletedFilesCount *int32 `avro:"deleted_data_files_count"`
+ AddedRowsCount *int64 `avro:"added_rows_count"`
+ ExistingRowsCount *int64 `avro:"existing_rows_count"`
+ DeletedRowsCount *int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV1) Version() int { return 1 }
+func (m *manifestFileV1) FilePath() string { return m.Path }
+func (m *manifestFileV1) Length() int64 { return m.Len }
+func (m *manifestFileV1) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV1) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV1) SnapshotID() int64 {
+ if m.AddedSnapshotID == nil {
+ return 0
+ }
+ return *m.AddedSnapshotID
+}
+
+func (m *manifestFileV1) AddedDataFiles() int32 {
+ if m.AddedFilesCount == nil {
+ return 0
+ }
+ return *m.AddedFilesCount
+}
+
+func (m *manifestFileV1) ExistingDataFiles() int32 {
+ if m.ExistingFilesCount == nil {
+ return 0
+ }
+ return *m.ExistingFilesCount
+}
+
+func (m *manifestFileV1) DeletedDataFiles() int32 {
+ if m.DeletedFilesCount == nil {
+ return 0
+ }
+ return *m.DeletedFilesCount
+}
+
+func (m *manifestFileV1) AddedRows() int64 {
+ if m.AddedRowsCount == nil {
+ return 0
+ }
+ return *m.AddedRowsCount
+}
+
+func (m *manifestFileV1) ExistingRows() int64 {
+ if m.ExistingRowsCount == nil {
+ return 0
+ }
+ return *m.ExistingRowsCount
+}
+
+func (m *manifestFileV1) DeletedRows() int64 {
+ if m.DeletedRowsCount == nil {
+ return 0
+ }
+ return *m.DeletedRowsCount
+}
+
+func (m *manifestFileV1) HasAddedFiles() bool {
+ return m.AddedFilesCount == nil || *m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV1) HasExistingFiles() bool {
+ return m.ExistingFilesCount == nil || *m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV1) SequenceNum() int64 { return 0 }
+func (m *manifestFileV1) MinSequenceNum() int64 { return 0 }
+func (m *manifestFileV1) Metadata() []byte { return m.KeyMetadata }
+func (m *manifestFileV1) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV1) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+type manifestFileV2 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ SeqNumber int64 `avro:"sequence_number"`
+ MinSeqNumber int64 `avro:"min_sequence_number"`
+ AddedSnapshotID int64 `avro:"added_snapshot_id"`
+ AddedFilesCount int32 `avro:"added_files_count"`
+ ExistingFilesCount int32 `avro:"existing_files_count"`
+ DeletedFilesCount int32 `avro:"deleted_files_count"`
+ AddedRowsCount int64 `avro:"added_rows_count"`
+ ExistingRowsCount int64 `avro:"existing_rows_count"`
+ DeletedRowsCount int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV2) Version() int { return 2 }
+
+func (m *manifestFileV2) FilePath() string { return m.Path }
+func (m *manifestFileV2) Length() int64 { return m.Len }
+func (m *manifestFileV2) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV2) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV2) SnapshotID() int64 {
+ return m.AddedSnapshotID
+}
+
+func (m *manifestFileV2) AddedDataFiles() int32 {
+ return m.AddedFilesCount
+}
+
+func (m *manifestFileV2) ExistingDataFiles() int32 {
+ return m.ExistingFilesCount
+}
+
+func (m *manifestFileV2) DeletedDataFiles() int32 {
+ return m.DeletedFilesCount
+}
+
+func (m *manifestFileV2) AddedRows() int64 {
+ return m.AddedRowsCount
+}
+
+func (m *manifestFileV2) ExistingRows() int64 {
+ return m.ExistingRowsCount
+}
+
+func (m *manifestFileV2) DeletedRows() int64 {
+ return m.DeletedRowsCount
+}
+
+func (m *manifestFileV2) SequenceNum() int64 { return m.SeqNumber }
+func (m *manifestFileV2) MinSequenceNum() int64 { return m.MinSeqNumber }
+func (m *manifestFileV2) Metadata() []byte { return m.KeyMetadata }
+
+func (m *manifestFileV2) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV2) HasAddedFiles() bool {
+ return m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV2) HasExistingFiles() bool {
+ return m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV2) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+func fetchManifestEntries(m ManifestFile, fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ f, err := fs.Open(m.FilePath())
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+
+ dec, err := ocf.NewDecoder(f)
+ if err != nil {
+ return nil, err
+ }
+
+ metadata := dec.Metadata()
+ isVer1 := true
+ if string(metadata["format-version"]) == "2" {
+ isVer1 = false
+ }
+
+ results := make([]ManifestEntry, 0)
+ for dec.HasNext() {
+ var tmp ManifestEntry
+ if isVer1 {
+ tmp = &manifestEntryV1{}
+ } else {
+ tmp = &manifestEntryV2{}
+ }
+
+ if err := dec.Decode(tmp); err != nil {
+ return nil, err
+ }
+
+ if !discardDeleted || tmp.Status() != EntryStatusDELETED {
+ tmp.inheritSeqNum(m)
+ results = append(results, tmp)
+ }
+ }
+
+ return results, dec.Error()
+}
+
+// ManifestFile is the interface which covers both V1 and V2 manifest files.
+type ManifestFile interface {
+ // Version returns the version number of this manifest file.
+ // It should be 1 or 2.
+ Version() int
+ // FilePath is the location URI of this manifest file.
+ FilePath() string
+ // Length is the length in bytes of the manifest file.
+ Length() int64
+ // PartitionID is the ID of the partition spec used to write
+ // this manifest. It must be listed in the table metadata
+ // partition-specs.
+ PartitionID() int32
+ // ManifestContent is the type of files tracked by this manifest,
+ // either data or delete files. All v1 manifests track data files.
+ ManifestContent() ManifestContent
+ // SnapshotID is the ID of the snapshot where this manifest file
+ // was added.
+ SnapshotID() int64
+ // AddedDataFiles returns the number of entries in the manifest that
+ // have the status of EntryStatusADDED.
+ AddedDataFiles() int32
+ // ExistingDataFiles returns the number of entries in the manifest
+ // which have the status of EntryStatusEXISTING.
+ ExistingDataFiles() int32
+ // DeletedDataFiles returns the number of entries in the manifest
+ // which have the status of EntryStatusDELETED.
+ DeletedDataFiles() int32
+ // AddedRows returns the number of rows in all files of the manifest
+ // that have status EntryStatusADDED.
+ AddedRows() int64
+ // ExistingRows returns the number of rows in all files of the manifest
+ // which have status EntryStatusEXISTING.
+ ExistingRows() int64
+ // DeletedRows returns the number of rows in all files of the manifest
+ // which have status EntryStatusDELETED.
+ DeletedRows() int64
+ // SequenceNum returns the sequence number when this manifest was
+ // added to the table. Will be 0 for v1 manifest lists.
+ SequenceNum() int64
+ // MinSequenceNum is the minimum data sequence number of all live data
+ // or delete files in the manifest. Will be 0 for v1 manifest lists.
+ MinSequenceNum() int64
+ // Metadata returns implementation-specific key metadata for encryption
+ // if it exists in the manifest list.
+ Metadata() []byte
+ // PartitionList returns a list of field summaries for each partition
+ // field in the spec. Each field in the list corresponds to a field in
+ // the manifest file's partition spec.
+ PartitionList() []fieldSummary
+
+ // HasAddedFiles returns true if AddedDataFiles > 0 or if it was null.
+ HasAddedFiles() bool
+ // HasExistingFiles returns true if ExistingDataFiles > 0 or if it was null.
+ HasExistingFiles() bool
+ // FetchEntries reads the manifest list file to fetch the list of
+ // manifest entries using the provided file system IO interface.
+ // If discardDeleted is true, entries for files containing deleted rows
+ // will be skipped.
+ FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error)
+}
+
+// ReadManifestList reads in an avro manifest list file and returns a slice
+// of manifest files or an error if one is encountered.
+func ReadManifestList(in io.Reader) ([]ManifestFile, error) {
+ dec, err := ocf.NewDecoder(in)
+ if err != nil {
+ return nil, err
+ }
+
+ out := make([]ManifestFile, 0)
+
+ for dec.HasNext() {
+ var file ManifestFile
+ if string(dec.Metadata()["format-version"]) == "2" {
+ file = &manifestFileV2{}
+ } else {
+ file = &manifestFileV1{}
+ }
+
+ if err := dec.Decode(file); err != nil {
+ return nil, err
+ }
+ out = append(out, file)
+ }
+
+ return out, dec.Error()
+}
+
+// ManifestEntryStatus defines constants for the entry status of
+// existing, added or deleted.
+type ManifestEntryStatus int8
+
+const (
+ EntryStatusEXISTING ManifestEntryStatus = 0
+ EntryStatusADDED ManifestEntryStatus = 1
+ EntryStatusDELETED ManifestEntryStatus = 2
+)
+
+// ManifestEntryContent defines constants for the type of file contents
+// in the file entries. Data, Position based deletes and equality based
+// deletes.
+type ManifestEntryContent int8
+
+const (
+ EntryContentData ManifestEntryContent = 0
+ EntryContentPosDeletes ManifestEntryContent = 1
+ EntryContentEqDeletes ManifestEntryContent = 2
+)
+
+// FileFormat defines constants for the format of data files.
+type FileFormat string
+
+const (
+ AvroFile FileFormat = "AVRO"
+ OrcFile FileFormat = "ORC"
+ ParquetFile FileFormat = "PARQUET"
+)
+
+type colMap[K, V any] struct {
+ Key K `avro:"key"`
+ Value V `avro:"value"`
+}
+
+func avroColMapToMap[K comparable, V any](c *[]colMap[K, V]) map[K]V {
+ if c == nil {
+ return nil
+ }
+
+ out := make(map[K]V)
+ for _, data := range *c {
+ out[data.Key] = data.Value
+ }
+ return out
+}
+
+type dataFile struct {
+ Content ManifestEntryContent `avro:"content"`
+ Path string `avro:"file_path"`
+ Format FileFormat `avro:"file_format"`
+ PartitionData map[string]any `avro:"partition"`
+ RecordCount int64 `avro:"record_count"`
+ FileSize int64 `avro:"file_size_in_bytes"`
+ BlockSizeInBytes int64 `avro:"block_size_in_bytes"`
+ ColSizes *[]colMap[int, int64] `avro:"column_sizes"`
+ ValCounts *[]colMap[int, int64] `avro:"value_counts"`
+ NullCounts *[]colMap[int, int64] `avro:"null_value_counts"`
+ NaNCounts *[]colMap[int, int64] `avro:"nan_value_counts"`
+ DistinctCounts *[]colMap[int, int64] `avro:"distinct_counts"`
+ LowerBounds *[]colMap[int, []byte] `avro:"lower_bounds"`
+ UpperBounds *[]colMap[int, []byte] `avro:"upper_bounds"`
+ KeyMetadata *[]byte `avro:"key_metadata"`
+ SplitOffsets *[]int64 `avro:"split_offsets"`
+ EqualityIDs *[]int `avro:"equality_ids"`
Review Comment:
yeah I just wanted to make sure we don't accidentally accept this field when we really shouldn't
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] zeroshade commented on pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade commented on PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#issuecomment-1696556960
@coded9 @Fokko @nastra @rdblue any chance for a review soon to get this in?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat(manifests): Adding implementation of manifest files [iceberg-go]
Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1355788936
##########
manifest.go:
##########
@@ -0,0 +1,655 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package iceberg
+
+import (
+ "io"
+ "sync"
+
+ iceio "github.com/apache/iceberg-go/io"
+
+ "github.com/hamba/avro/v2/ocf"
+)
+
+// ManifestContent indicates the type of data inside of the files
+// described by a manifest. This will indicate whether the data files
+// contain active data or deleted rows.
+type ManifestContent int32
+
+const (
+ ManifestContentData ManifestContent = 0
+ ManifestContentDeletes ManifestContent = 1
+)
+
+type fieldSummary struct {
+ ContainsNull bool `avro:"contains_null"`
+ ContainsNaN *bool `avro:"contains_nan"`
+ LowerBound *[]byte `avro:"lower_bound"`
+ UpperBound *[]byte `avro:"upper_bound"`
+}
+
+type manifestFileV1 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
Review Comment:
sure thing! i'll make a PR for that, thanks for the heads up
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] nastra commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1326982917
##########
manifest.go:
##########
@@ -0,0 +1,655 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package iceberg
+
+import (
+ "io"
+ "sync"
+
+ iceio "github.com/apache/iceberg-go/io"
+
+ "github.com/hamba/avro/v2/ocf"
+)
+
+// ManifestContent indicates the type of data inside of the files
+// described by a manifest. This will indicate whether the data files
+// contain active data or deleted rows.
+type ManifestContent int32
+
+const (
+ ManifestContentData ManifestContent = 0
+ ManifestContentDeletes ManifestContent = 1
+)
+
+type fieldSummary struct {
+ ContainsNull bool `avro:"contains_null"`
+ ContainsNaN *bool `avro:"contains_nan"`
+ LowerBound *[]byte `avro:"lower_bound"`
+ UpperBound *[]byte `avro:"upper_bound"`
+}
+
+type manifestFileV1 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ AddedSnapshotID *int64 `avro:"added_snapshot_id"`
+ AddedFilesCount *int32 `avro:"added_data_files_count"`
+ ExistingFilesCount *int32 `avro:"existing_data_files_count"`
+ DeletedFilesCount *int32 `avro:"deleted_data_files_count"`
+ AddedRowsCount *int64 `avro:"added_rows_count"`
+ ExistingRowsCount *int64 `avro:"existing_rows_count"`
+ DeletedRowsCount *int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV1) Version() int { return 1 }
+func (m *manifestFileV1) FilePath() string { return m.Path }
+func (m *manifestFileV1) Length() int64 { return m.Len }
+func (m *manifestFileV1) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV1) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV1) SnapshotID() int64 {
+ if m.AddedSnapshotID == nil {
+ return 0
+ }
+ return *m.AddedSnapshotID
+}
+
+func (m *manifestFileV1) AddedDataFiles() int32 {
+ if m.AddedFilesCount == nil {
+ return 0
+ }
+ return *m.AddedFilesCount
+}
+
+func (m *manifestFileV1) ExistingDataFiles() int32 {
+ if m.ExistingFilesCount == nil {
+ return 0
+ }
+ return *m.ExistingFilesCount
+}
+
+func (m *manifestFileV1) DeletedDataFiles() int32 {
+ if m.DeletedFilesCount == nil {
+ return 0
+ }
+ return *m.DeletedFilesCount
+}
+
+func (m *manifestFileV1) AddedRows() int64 {
+ if m.AddedRowsCount == nil {
+ return 0
+ }
+ return *m.AddedRowsCount
+}
+
+func (m *manifestFileV1) ExistingRows() int64 {
+ if m.ExistingRowsCount == nil {
+ return 0
+ }
+ return *m.ExistingRowsCount
+}
+
+func (m *manifestFileV1) DeletedRows() int64 {
+ if m.DeletedRowsCount == nil {
+ return 0
+ }
+ return *m.DeletedRowsCount
+}
+
+func (m *manifestFileV1) HasAddedFiles() bool {
+ return m.AddedFilesCount == nil || *m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV1) HasExistingFiles() bool {
+ return m.ExistingFilesCount == nil || *m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV1) SequenceNum() int64 { return 0 }
+func (m *manifestFileV1) MinSequenceNum() int64 { return 0 }
+func (m *manifestFileV1) Metadata() []byte { return m.KeyMetadata }
+func (m *manifestFileV1) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV1) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+type manifestFileV2 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ SeqNumber int64 `avro:"sequence_number"`
+ MinSeqNumber int64 `avro:"min_sequence_number"`
+ AddedSnapshotID int64 `avro:"added_snapshot_id"`
+ AddedFilesCount int32 `avro:"added_files_count"`
+ ExistingFilesCount int32 `avro:"existing_files_count"`
+ DeletedFilesCount int32 `avro:"deleted_files_count"`
+ AddedRowsCount int64 `avro:"added_rows_count"`
+ ExistingRowsCount int64 `avro:"existing_rows_count"`
+ DeletedRowsCount int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV2) Version() int { return 2 }
+
+func (m *manifestFileV2) FilePath() string { return m.Path }
+func (m *manifestFileV2) Length() int64 { return m.Len }
+func (m *manifestFileV2) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV2) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV2) SnapshotID() int64 {
+ return m.AddedSnapshotID
+}
+
+func (m *manifestFileV2) AddedDataFiles() int32 {
+ return m.AddedFilesCount
+}
+
+func (m *manifestFileV2) ExistingDataFiles() int32 {
+ return m.ExistingFilesCount
+}
+
+func (m *manifestFileV2) DeletedDataFiles() int32 {
+ return m.DeletedFilesCount
+}
+
+func (m *manifestFileV2) AddedRows() int64 {
+ return m.AddedRowsCount
+}
+
+func (m *manifestFileV2) ExistingRows() int64 {
+ return m.ExistingRowsCount
+}
+
+func (m *manifestFileV2) DeletedRows() int64 {
+ return m.DeletedRowsCount
+}
+
+func (m *manifestFileV2) SequenceNum() int64 { return m.SeqNumber }
+func (m *manifestFileV2) MinSequenceNum() int64 { return m.MinSeqNumber }
+func (m *manifestFileV2) Metadata() []byte { return m.KeyMetadata }
+
+func (m *manifestFileV2) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV2) HasAddedFiles() bool {
+ return m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV2) HasExistingFiles() bool {
+ return m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV2) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+func fetchManifestEntries(m ManifestFile, fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ f, err := fs.Open(m.FilePath())
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+
+ dec, err := ocf.NewDecoder(f)
+ if err != nil {
+ return nil, err
+ }
+
+ metadata := dec.Metadata()
+ isVer1 := true
+ if string(metadata["format-version"]) == "2" {
+ isVer1 = false
+ }
+
+ results := make([]ManifestEntry, 0)
+ for dec.HasNext() {
+ var tmp ManifestEntry
+ if isVer1 {
+ tmp = &manifestEntryV1{}
+ } else {
+ tmp = &manifestEntryV2{}
+ }
+
+ if err := dec.Decode(tmp); err != nil {
+ return nil, err
+ }
+
+ if !discardDeleted || tmp.Status() != EntryStatusDELETED {
+ tmp.inheritSeqNum(m)
+ results = append(results, tmp)
+ }
+ }
+
+ return results, dec.Error()
+}
+
+// ManifestFile is the interface which covers both V1 and V2 manifest files.
+type ManifestFile interface {
+ // Version returns the version number of this manifest file.
+ // It should be 1 or 2.
+ Version() int
+ // FilePath is the location URI of this manifest file.
+ FilePath() string
+ // Length is the length in bytes of the manifest file.
+ Length() int64
+ // PartitionID is the ID of the partition spec used to write
+ // this manifest. It must be listed in the table metadata
+ // partition-specs.
+ PartitionID() int32
+ // ManifestContent is the type of files tracked by this manifest,
+ // either data or delete files. All v1 manifests track data files.
+ ManifestContent() ManifestContent
+ // SnapshotID is the ID of the snapshot where this manifest file
+ // was added.
+ SnapshotID() int64
+ // AddedDataFiles returns the number of entries in the manifest that
+ // have the status of EntryStatusADDED.
+ AddedDataFiles() int32
+ // ExistingDataFiles returns the number of entries in the manifest
+ // which have the status of EntryStatusEXISTING.
+ ExistingDataFiles() int32
+ // DeletedDataFiles returns the number of entries in the manifest
+ // which have the status of EntryStatusDELETED.
+ DeletedDataFiles() int32
+ // AddedRows returns the number of rows in all files of the manifest
+ // that have status EntryStatusADDED.
+ AddedRows() int64
+ // ExistingRows returns the number of rows in all files of the manifest
+ // which have status EntryStatusEXISTING.
+ ExistingRows() int64
+ // DeletedRows returns the number of rows in all files of the manifest
+ // which have status EntryStatusDELETED.
+ DeletedRows() int64
+ // SequenceNum returns the sequence number when this manifest was
+ // added to the table. Will be 0 for v1 manifest lists.
+ SequenceNum() int64
+ // MinSequenceNum is the minimum data sequence number of all live data
+ // or delete files in the manifest. Will be 0 for v1 manifest lists.
+ MinSequenceNum() int64
+ // Metadata returns implementation-specific key metadata for encryption
+ // if it exists in the manifest list.
+ Metadata() []byte
+ // PartitionList returns a list of field summaries for each partition
+ // field in the spec. Each field in the list corresponds to a field in
+ // the manifest file's partition spec.
+ PartitionList() []fieldSummary
Review Comment:
maybe just `Partitions()`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] nastra commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1326964804
##########
manifest.go:
##########
@@ -0,0 +1,655 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package iceberg
+
+import (
+ "io"
+ "sync"
+
+ iceio "github.com/apache/iceberg-go/io"
+
+ "github.com/hamba/avro/v2/ocf"
+)
+
+// ManifestContent indicates the type of data inside of the files
+// described by a manifest. This will indicate whether the data files
+// contain active data or deleted rows.
+type ManifestContent int32
+
+const (
+ ManifestContentData ManifestContent = 0
+ ManifestContentDeletes ManifestContent = 1
+)
+
+type fieldSummary struct {
+ ContainsNull bool `avro:"contains_null"`
+ ContainsNaN *bool `avro:"contains_nan"`
+ LowerBound *[]byte `avro:"lower_bound"`
+ UpperBound *[]byte `avro:"upper_bound"`
+}
+
+type manifestFileV1 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ AddedSnapshotID *int64 `avro:"added_snapshot_id"`
+ AddedFilesCount *int32 `avro:"added_data_files_count"`
+ ExistingFilesCount *int32 `avro:"existing_data_files_count"`
+ DeletedFilesCount *int32 `avro:"deleted_data_files_count"`
+ AddedRowsCount *int64 `avro:"added_rows_count"`
+ ExistingRowsCount *int64 `avro:"existing_rows_count"`
+ DeletedRowsCount *int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV1) Version() int { return 1 }
+func (m *manifestFileV1) FilePath() string { return m.Path }
+func (m *manifestFileV1) Length() int64 { return m.Len }
+func (m *manifestFileV1) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV1) ManifestContent() ManifestContent { return m.Content }
Review Comment:
can be removed, because content isn't part of a v1 manifest file
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] zeroshade commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1328879119
##########
manifest.go:
##########
@@ -0,0 +1,655 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package iceberg
+
+import (
+ "io"
+ "sync"
+
+ iceio "github.com/apache/iceberg-go/io"
+
+ "github.com/hamba/avro/v2/ocf"
+)
+
+// ManifestContent indicates the type of data inside of the files
+// described by a manifest. This will indicate whether the data files
+// contain active data or deleted rows.
+type ManifestContent int32
+
+const (
+ ManifestContentData ManifestContent = 0
+ ManifestContentDeletes ManifestContent = 1
+)
+
+type fieldSummary struct {
+ ContainsNull bool `avro:"contains_null"`
+ ContainsNaN *bool `avro:"contains_nan"`
+ LowerBound *[]byte `avro:"lower_bound"`
+ UpperBound *[]byte `avro:"upper_bound"`
+}
+
+type manifestFileV1 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ AddedSnapshotID *int64 `avro:"added_snapshot_id"`
+ AddedFilesCount *int32 `avro:"added_data_files_count"`
+ ExistingFilesCount *int32 `avro:"existing_data_files_count"`
+ DeletedFilesCount *int32 `avro:"deleted_data_files_count"`
+ AddedRowsCount *int64 `avro:"added_rows_count"`
+ ExistingRowsCount *int64 `avro:"existing_rows_count"`
+ DeletedRowsCount *int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV1) Version() int { return 1 }
+func (m *manifestFileV1) FilePath() string { return m.Path }
+func (m *manifestFileV1) Length() int64 { return m.Len }
+func (m *manifestFileV1) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV1) ManifestContent() ManifestContent { return m.Content }
Review Comment:
The function needs to exist so that it meets the interface, but since `Content` doesn't exist in v1, i'll just hardcode it to return `ManifestContentData`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] nastra commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1331873382
##########
manifest.go:
##########
@@ -0,0 +1,655 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package iceberg
+
+import (
+ "io"
+ "sync"
+
+ iceio "github.com/apache/iceberg-go/io"
+
+ "github.com/hamba/avro/v2/ocf"
+)
+
+// ManifestContent indicates the type of data inside of the files
+// described by a manifest. This will indicate whether the data files
+// contain active data or deleted rows.
+type ManifestContent int32
+
+const (
+ ManifestContentData ManifestContent = 0
+ ManifestContentDeletes ManifestContent = 1
+)
+
+type fieldSummary struct {
+ ContainsNull bool `avro:"contains_null"`
+ ContainsNaN *bool `avro:"contains_nan"`
+ LowerBound *[]byte `avro:"lower_bound"`
+ UpperBound *[]byte `avro:"upper_bound"`
+}
+
+type manifestFileV1 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ AddedSnapshotID *int64 `avro:"added_snapshot_id"`
+ AddedFilesCount *int32 `avro:"added_data_files_count"`
+ ExistingFilesCount *int32 `avro:"existing_data_files_count"`
+ DeletedFilesCount *int32 `avro:"deleted_data_files_count"`
+ AddedRowsCount *int64 `avro:"added_rows_count"`
+ ExistingRowsCount *int64 `avro:"existing_rows_count"`
+ DeletedRowsCount *int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV1) Version() int { return 1 }
+func (m *manifestFileV1) FilePath() string { return m.Path }
+func (m *manifestFileV1) Length() int64 { return m.Len }
+func (m *manifestFileV1) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV1) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV1) SnapshotID() int64 {
+ if m.AddedSnapshotID == nil {
+ return 0
+ }
+ return *m.AddedSnapshotID
+}
+
+func (m *manifestFileV1) AddedDataFiles() int32 {
+ if m.AddedFilesCount == nil {
+ return 0
+ }
+ return *m.AddedFilesCount
+}
+
+func (m *manifestFileV1) ExistingDataFiles() int32 {
+ if m.ExistingFilesCount == nil {
+ return 0
+ }
+ return *m.ExistingFilesCount
+}
+
+func (m *manifestFileV1) DeletedDataFiles() int32 {
+ if m.DeletedFilesCount == nil {
+ return 0
+ }
+ return *m.DeletedFilesCount
+}
+
+func (m *manifestFileV1) AddedRows() int64 {
+ if m.AddedRowsCount == nil {
+ return 0
+ }
+ return *m.AddedRowsCount
+}
+
+func (m *manifestFileV1) ExistingRows() int64 {
+ if m.ExistingRowsCount == nil {
+ return 0
+ }
+ return *m.ExistingRowsCount
+}
+
+func (m *manifestFileV1) DeletedRows() int64 {
+ if m.DeletedRowsCount == nil {
+ return 0
+ }
+ return *m.DeletedRowsCount
+}
+
+func (m *manifestFileV1) HasAddedFiles() bool {
+ return m.AddedFilesCount == nil || *m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV1) HasExistingFiles() bool {
+ return m.ExistingFilesCount == nil || *m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV1) SequenceNum() int64 { return 0 }
+func (m *manifestFileV1) MinSequenceNum() int64 { return 0 }
+func (m *manifestFileV1) Metadata() []byte { return m.KeyMetadata }
+func (m *manifestFileV1) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV1) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+type manifestFileV2 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ SeqNumber int64 `avro:"sequence_number"`
+ MinSeqNumber int64 `avro:"min_sequence_number"`
+ AddedSnapshotID int64 `avro:"added_snapshot_id"`
+ AddedFilesCount int32 `avro:"added_files_count"`
+ ExistingFilesCount int32 `avro:"existing_files_count"`
+ DeletedFilesCount int32 `avro:"deleted_files_count"`
+ AddedRowsCount int64 `avro:"added_rows_count"`
+ ExistingRowsCount int64 `avro:"existing_rows_count"`
+ DeletedRowsCount int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV2) Version() int { return 2 }
+
+func (m *manifestFileV2) FilePath() string { return m.Path }
+func (m *manifestFileV2) Length() int64 { return m.Len }
+func (m *manifestFileV2) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV2) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV2) SnapshotID() int64 {
+ return m.AddedSnapshotID
+}
+
+func (m *manifestFileV2) AddedDataFiles() int32 {
+ return m.AddedFilesCount
+}
+
+func (m *manifestFileV2) ExistingDataFiles() int32 {
+ return m.ExistingFilesCount
+}
+
+func (m *manifestFileV2) DeletedDataFiles() int32 {
+ return m.DeletedFilesCount
+}
+
+func (m *manifestFileV2) AddedRows() int64 {
+ return m.AddedRowsCount
+}
+
+func (m *manifestFileV2) ExistingRows() int64 {
+ return m.ExistingRowsCount
+}
+
+func (m *manifestFileV2) DeletedRows() int64 {
+ return m.DeletedRowsCount
+}
+
+func (m *manifestFileV2) SequenceNum() int64 { return m.SeqNumber }
+func (m *manifestFileV2) MinSequenceNum() int64 { return m.MinSeqNumber }
+func (m *manifestFileV2) Metadata() []byte { return m.KeyMetadata }
+
+func (m *manifestFileV2) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV2) HasAddedFiles() bool {
+ return m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV2) HasExistingFiles() bool {
+ return m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV2) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+func fetchManifestEntries(m ManifestFile, fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ f, err := fs.Open(m.FilePath())
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+
+ dec, err := ocf.NewDecoder(f)
+ if err != nil {
+ return nil, err
+ }
+
+ metadata := dec.Metadata()
+ isVer1 := true
+ if string(metadata["format-version"]) == "2" {
+ isVer1 = false
+ }
+
+ results := make([]ManifestEntry, 0)
+ for dec.HasNext() {
+ var tmp ManifestEntry
+ if isVer1 {
+ tmp = &manifestEntryV1{}
+ } else {
+ tmp = &manifestEntryV2{}
+ }
+
+ if err := dec.Decode(tmp); err != nil {
+ return nil, err
+ }
+
+ if !discardDeleted || tmp.Status() != EntryStatusDELETED {
+ tmp.inheritSeqNum(m)
+ results = append(results, tmp)
+ }
+ }
+
+ return results, dec.Error()
+}
+
+// ManifestFile is the interface which covers both V1 and V2 manifest files.
+type ManifestFile interface {
+ // Version returns the version number of this manifest file.
+ // It should be 1 or 2.
+ Version() int
+ // FilePath is the location URI of this manifest file.
+ FilePath() string
+ // Length is the length in bytes of the manifest file.
+ Length() int64
+ // PartitionID is the ID of the partition spec used to write
+ // this manifest. It must be listed in the table metadata
+ // partition-specs.
+ PartitionID() int32
+ // ManifestContent is the type of files tracked by this manifest,
+ // either data or delete files. All v1 manifests track data files.
+ ManifestContent() ManifestContent
+ // SnapshotID is the ID of the snapshot where this manifest file
+ // was added.
+ SnapshotID() int64
+ // AddedDataFiles returns the number of entries in the manifest that
+ // have the status of EntryStatusADDED.
+ AddedDataFiles() int32
+ // ExistingDataFiles returns the number of entries in the manifest
+ // which have the status of EntryStatusEXISTING.
+ ExistingDataFiles() int32
+ // DeletedDataFiles returns the number of entries in the manifest
+ // which have the status of EntryStatusDELETED.
+ DeletedDataFiles() int32
+ // AddedRows returns the number of rows in all files of the manifest
+ // that have status EntryStatusADDED.
+ AddedRows() int64
+ // ExistingRows returns the number of rows in all files of the manifest
+ // which have status EntryStatusEXISTING.
+ ExistingRows() int64
+ // DeletedRows returns the number of rows in all files of the manifest
+ // which have status EntryStatusDELETED.
+ DeletedRows() int64
+ // SequenceNum returns the sequence number when this manifest was
+ // added to the table. Will be 0 for v1 manifest lists.
+ SequenceNum() int64
+ // MinSequenceNum is the minimum data sequence number of all live data
+ // or delete files in the manifest. Will be 0 for v1 manifest lists.
+ MinSequenceNum() int64
+ // Metadata returns implementation-specific key metadata for encryption
+ // if it exists in the manifest list.
+ Metadata() []byte
+ // PartitionList returns a list of field summaries for each partition
+ // field in the spec. Each field in the list corresponds to a field in
+ // the manifest file's partition spec.
+ PartitionList() []fieldSummary
+
+ // HasAddedFiles returns true if AddedDataFiles > 0 or if it was null.
+ HasAddedFiles() bool
+ // HasExistingFiles returns true if ExistingDataFiles > 0 or if it was null.
+ HasExistingFiles() bool
+ // FetchEntries reads the manifest list file to fetch the list of
+ // manifest entries using the provided file system IO interface.
+ // If discardDeleted is true, entries for files containing deleted rows
+ // will be skipped.
+ FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error)
+}
+
+// ReadManifestList reads in an avro manifest list file and returns a slice
+// of manifest files or an error if one is encountered.
+func ReadManifestList(in io.Reader) ([]ManifestFile, error) {
+ dec, err := ocf.NewDecoder(in)
+ if err != nil {
+ return nil, err
+ }
+
+ out := make([]ManifestFile, 0)
+
+ for dec.HasNext() {
+ var file ManifestFile
+ if string(dec.Metadata()["format-version"]) == "2" {
+ file = &manifestFileV2{}
+ } else {
+ file = &manifestFileV1{}
+ }
+
+ if err := dec.Decode(file); err != nil {
+ return nil, err
+ }
+ out = append(out, file)
+ }
+
+ return out, dec.Error()
+}
+
+// ManifestEntryStatus defines constants for the entry status of
+// existing, added or deleted.
+type ManifestEntryStatus int8
+
+const (
+ EntryStatusEXISTING ManifestEntryStatus = 0
+ EntryStatusADDED ManifestEntryStatus = 1
+ EntryStatusDELETED ManifestEntryStatus = 2
+)
+
+// ManifestEntryContent defines constants for the type of file contents
+// in the file entries. Data, Position based deletes and equality based
+// deletes.
+type ManifestEntryContent int8
+
+const (
+ EntryContentData ManifestEntryContent = 0
+ EntryContentPosDeletes ManifestEntryContent = 1
+ EntryContentEqDeletes ManifestEntryContent = 2
+)
+
+// FileFormat defines constants for the format of data files.
+type FileFormat string
+
+const (
+ AvroFile FileFormat = "AVRO"
+ OrcFile FileFormat = "ORC"
+ ParquetFile FileFormat = "PARQUET"
+)
+
+type colMap[K, V any] struct {
+ Key K `avro:"key"`
+ Value V `avro:"value"`
+}
+
+func avroColMapToMap[K comparable, V any](c *[]colMap[K, V]) map[K]V {
+ if c == nil {
+ return nil
+ }
+
+ out := make(map[K]V)
+ for _, data := range *c {
+ out[data.Key] = data.Value
+ }
+ return out
+}
+
+type dataFile struct {
+ Content ManifestEntryContent `avro:"content"`
Review Comment:
readers would/should ignore it. Would it cause too much trouble to not write this field?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] nastra commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1331920399
##########
manifest.go:
##########
@@ -0,0 +1,655 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package iceberg
+
+import (
+ "io"
+ "sync"
+
+ iceio "github.com/apache/iceberg-go/io"
+
+ "github.com/hamba/avro/v2/ocf"
+)
+
+// ManifestContent indicates the type of data inside of the files
+// described by a manifest. This will indicate whether the data files
+// contain active data or deleted rows.
+type ManifestContent int32
+
+const (
+ ManifestContentData ManifestContent = 0
+ ManifestContentDeletes ManifestContent = 1
+)
+
+type fieldSummary struct {
+ ContainsNull bool `avro:"contains_null"`
+ ContainsNaN *bool `avro:"contains_nan"`
+ LowerBound *[]byte `avro:"lower_bound"`
+ UpperBound *[]byte `avro:"upper_bound"`
+}
+
+type manifestFileV1 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
Review Comment:
I've opened https://github.com/apache/iceberg/pull/8600 to fix the Spec
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] zeroshade commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1326140649
##########
io/io.go:
##########
@@ -0,0 +1,327 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package io
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "io/fs"
+ "net/http"
+ "net/url"
+ "os"
+ "strings"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
+ "github.com/aws/aws-sdk-go-v2/config"
+ "github.com/aws/aws-sdk-go-v2/credentials"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+ "github.com/wolfeidau/s3iofs"
+)
+
+const (
+ S3RegionName = "s3.region-name"
+ S3SessionToken = "s3.session-token"
+ S3SecretAccessKey = "s3.secret-access-key"
+ S3AccessKeyID = "s3.access-key-id"
+ S3EndpointURL = "s3.region"
+ S3ProxyURI = "s3.proxy-uri"
+)
+
+// IO is an interface to a hierarchical file system.
+//
+// The IO interface is the minimum implementation required for a file
+// system to utilize an iceberg table. A file system may implement
+// additional interfaces, such as ReadFileIO, to provide additional or
+// optimized functionality.
+type IO interface {
+ // Open opens the named file.
+ //
+ // When Open returns an error, it should be of type *PathError
+ // with the Op field set to "open", the Path field set to name,
+ // and the Err field describing the problem.
+ //
+ // Open should reject attempts to open names that do not satisfy
+ // fs.ValidPath(name), returning a *PathError with Err set to
+ // ErrInvalid or ErrNotExist.
+ Open(name string) (File, error)
+
+ // Remove removes the named file or (empty) directory.
+ //
+ // If there is an error, it will be of type *PathError.
+ Remove(name string) error
+}
+
+// ReadFileIO is the interface implemented by a file system that
+// provides an optimized implementation of ReadFile.
+type ReadFileIO interface {
+ IO
+
+ // ReadFile reads the named file and returns its contents.
+ // A successful call returns a nil error, not io.EOF.
+ // (Because ReadFile reads the whole file, the expected EOF
+ // from the final Read is not treated as an error to be reported.)
+ //
+ // The caller is permitted to modify the returned byte slice.
+ // This method should return a copy of the underlying data.
+ ReadFile(name string) ([]byte, error)
+}
+
+// A File provides access to a single file. The File interface is the
+// minimum implementation required for Iceberg to interact with a file.
+// Directory files should also implement
+type File interface {
+ fs.File
+ io.ReadSeekCloser
+ io.ReaderAt
+}
+
+// A ReadDirFile is a directory file whose entries can be read with the
+// ReadDir method. Every directory file should implement this interface.
+// (It is permissible for any file to implement this interface, but
+// if so ReadDir should return an error for non-directories.)
+type ReadDirFile interface {
+ File
+
+ // ReadDir read the contents of the directory and returns a slice
+ // of up to n DirEntry values in directory order. Subsequent calls
+ // on the same file will yield further DirEntry values.
+ //
+ // If n > 0, ReadDir returns at most n DirEntry structures. In this
+ // case, if ReadDir returns an empty slice, it will return a non-nil
+ // error explaining why.
+ //
+ // At the end of a directory, the error is io.EOF. (ReadDir must return
+ // io.EOF itself, not an error wrapping io.EOF.)
+ //
+ // If n <= 0, ReadDir returns all the DirEntry values from the directory
+ // in a single slice. In this case, if ReadDir succeeds (reads all the way
+ // to the end of the directory), it returns the slice and a nil error.
+ // If it encounters an error before the end of the directory, ReadDir
+ // returns the DirEntry list read until that point and a non-nil error.
+ ReadDir(n int) ([]fs.DirEntry, error)
+}
+
+// FS wraps an io/fs.FS as an IO interface.
+func FS(fsys fs.FS) IO {
+ if _, ok := fsys.(fs.ReadFileFS); ok {
+ return readFileFS{ioFS{fsys, nil}}
+ }
+ return ioFS{fsys, nil}
+}
+
+// FSPreProcName wraps an io/fs.FS like FS, only if fn is non-nil then
+// it is called to preprocess any filenames before they are passed to
+// the underlying fsys.
+func FSPreProcName(fsys fs.FS, fn func(string) string) IO {
+ if _, ok := fsys.(fs.ReadFileFS); ok {
+ return readFileFS{ioFS{fsys, fn}}
+ }
+ return ioFS{fsys, fn}
+}
+
+type readFileFS struct {
+ ioFS
+}
+
+func (r readFileFS) ReadFile(name string) ([]byte, error) {
+ if r.preProcessName != nil {
+ name = r.preProcessName(name)
+ }
+
+ rfs, ok := r.fsys.(fs.ReadFileFS)
+ if !ok {
+ return nil, errMissingReadFile
+ }
+ return rfs.ReadFile(name)
+}
+
+type ioFS struct {
+ fsys fs.FS
+
+ preProcessName func(string) string
+}
+
+func (f ioFS) Open(name string) (File, error) {
+ if f.preProcessName != nil {
+ name = f.preProcessName(name)
+ }
+
+ if name == "/" {
+ name = "."
+ } else {
+ name = strings.TrimPrefix(name, "/")
+ }
+ file, err := f.fsys.Open(name)
+ if err != nil {
+ return nil, err
+ }
+
+ return ioFile{file}, nil
+}
+
+func (f ioFS) Remove(name string) error {
+ r, ok := f.fsys.(interface{ Remove(name string) error })
+ if !ok {
+ return errMissingRemove
+ }
+ return r.Remove(name)
+}
+
+var (
+ errMissingReadDir = errors.New("fs.File directory missing ReadDir method")
+ errMissingSeek = errors.New("fs.File missing Seek method")
+ errMissingReadAt = errors.New("fs.File missing ReadAt")
+ errMissingRemove = errors.New("fs.FS missing Remove method")
+ errMissingReadFile = errors.New("fs.FS missing ReadFile method")
+)
+
+type ioFile struct {
+ file fs.File
+}
+
+func (f ioFile) Close() error { return f.file.Close() }
+func (f ioFile) Read(b []byte) (int, error) { return f.file.Read(b) }
+func (f ioFile) Stat() (fs.FileInfo, error) { return f.file.Stat() }
+func (f ioFile) Seek(offset int64, whence int) (int64, error) {
+ s, ok := f.file.(io.Seeker)
+ if !ok {
+ return 0, errMissingSeek
+ }
+ return s.Seek(offset, whence)
+}
+
+func (f ioFile) ReadAt(p []byte, off int64) (n int, err error) {
+ r, ok := f.file.(io.ReaderAt)
+ if !ok {
+ return 0, errMissingReadAt
+ }
+ return r.ReadAt(p, off)
+}
+
+func (f ioFile) ReadDir(count int) ([]fs.DirEntry, error) {
+ d, ok := f.file.(fs.ReadDirFile)
+ if !ok {
+ return nil, errMissingReadDir
+ }
+
+ return d.ReadDir(count)
+}
+
+func inferFileIOFromSchema(path string, props map[string]string) (IO, error) {
+ parsed, err := url.Parse(path)
+ if err != nil {
+ return nil, err
+ }
+
+ switch parsed.Scheme {
+ case "s3", "s3a", "s3n":
+ opts := []func(*config.LoadOptions) error{}
+ endpoint, ok := props[S3EndpointURL]
+ if !ok {
+ endpoint = os.Getenv("AWS_S3_ENDPOINT")
+ }
+
+ if endpoint != "" {
Review Comment:
That's a fair point. I'll update this shortly
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] nastra commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1326997475
##########
manifest.go:
##########
@@ -0,0 +1,655 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package iceberg
+
+import (
+ "io"
+ "sync"
+
+ iceio "github.com/apache/iceberg-go/io"
+
+ "github.com/hamba/avro/v2/ocf"
+)
+
+// ManifestContent indicates the type of data inside of the files
+// described by a manifest. This will indicate whether the data files
+// contain active data or deleted rows.
+type ManifestContent int32
+
+const (
+ ManifestContentData ManifestContent = 0
+ ManifestContentDeletes ManifestContent = 1
+)
+
+type fieldSummary struct {
+ ContainsNull bool `avro:"contains_null"`
+ ContainsNaN *bool `avro:"contains_nan"`
+ LowerBound *[]byte `avro:"lower_bound"`
+ UpperBound *[]byte `avro:"upper_bound"`
+}
+
+type manifestFileV1 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ AddedSnapshotID *int64 `avro:"added_snapshot_id"`
+ AddedFilesCount *int32 `avro:"added_data_files_count"`
+ ExistingFilesCount *int32 `avro:"existing_data_files_count"`
+ DeletedFilesCount *int32 `avro:"deleted_data_files_count"`
+ AddedRowsCount *int64 `avro:"added_rows_count"`
+ ExistingRowsCount *int64 `avro:"existing_rows_count"`
+ DeletedRowsCount *int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV1) Version() int { return 1 }
+func (m *manifestFileV1) FilePath() string { return m.Path }
+func (m *manifestFileV1) Length() int64 { return m.Len }
+func (m *manifestFileV1) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV1) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV1) SnapshotID() int64 {
+ if m.AddedSnapshotID == nil {
+ return 0
+ }
+ return *m.AddedSnapshotID
+}
+
+func (m *manifestFileV1) AddedDataFiles() int32 {
+ if m.AddedFilesCount == nil {
+ return 0
+ }
+ return *m.AddedFilesCount
+}
+
+func (m *manifestFileV1) ExistingDataFiles() int32 {
+ if m.ExistingFilesCount == nil {
+ return 0
+ }
+ return *m.ExistingFilesCount
+}
+
+func (m *manifestFileV1) DeletedDataFiles() int32 {
+ if m.DeletedFilesCount == nil {
+ return 0
+ }
+ return *m.DeletedFilesCount
+}
+
+func (m *manifestFileV1) AddedRows() int64 {
+ if m.AddedRowsCount == nil {
+ return 0
+ }
+ return *m.AddedRowsCount
+}
+
+func (m *manifestFileV1) ExistingRows() int64 {
+ if m.ExistingRowsCount == nil {
+ return 0
+ }
+ return *m.ExistingRowsCount
+}
+
+func (m *manifestFileV1) DeletedRows() int64 {
+ if m.DeletedRowsCount == nil {
+ return 0
+ }
+ return *m.DeletedRowsCount
+}
+
+func (m *manifestFileV1) HasAddedFiles() bool {
+ return m.AddedFilesCount == nil || *m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV1) HasExistingFiles() bool {
+ return m.ExistingFilesCount == nil || *m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV1) SequenceNum() int64 { return 0 }
+func (m *manifestFileV1) MinSequenceNum() int64 { return 0 }
+func (m *manifestFileV1) Metadata() []byte { return m.KeyMetadata }
+func (m *manifestFileV1) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV1) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+type manifestFileV2 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ SeqNumber int64 `avro:"sequence_number"`
+ MinSeqNumber int64 `avro:"min_sequence_number"`
+ AddedSnapshotID int64 `avro:"added_snapshot_id"`
+ AddedFilesCount int32 `avro:"added_files_count"`
+ ExistingFilesCount int32 `avro:"existing_files_count"`
+ DeletedFilesCount int32 `avro:"deleted_files_count"`
+ AddedRowsCount int64 `avro:"added_rows_count"`
+ ExistingRowsCount int64 `avro:"existing_rows_count"`
+ DeletedRowsCount int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV2) Version() int { return 2 }
+
+func (m *manifestFileV2) FilePath() string { return m.Path }
+func (m *manifestFileV2) Length() int64 { return m.Len }
+func (m *manifestFileV2) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV2) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV2) SnapshotID() int64 {
+ return m.AddedSnapshotID
+}
+
+func (m *manifestFileV2) AddedDataFiles() int32 {
+ return m.AddedFilesCount
+}
+
+func (m *manifestFileV2) ExistingDataFiles() int32 {
+ return m.ExistingFilesCount
+}
+
+func (m *manifestFileV2) DeletedDataFiles() int32 {
+ return m.DeletedFilesCount
+}
+
+func (m *manifestFileV2) AddedRows() int64 {
+ return m.AddedRowsCount
+}
+
+func (m *manifestFileV2) ExistingRows() int64 {
+ return m.ExistingRowsCount
+}
+
+func (m *manifestFileV2) DeletedRows() int64 {
+ return m.DeletedRowsCount
+}
+
+func (m *manifestFileV2) SequenceNum() int64 { return m.SeqNumber }
+func (m *manifestFileV2) MinSequenceNum() int64 { return m.MinSeqNumber }
+func (m *manifestFileV2) Metadata() []byte { return m.KeyMetadata }
+
+func (m *manifestFileV2) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV2) HasAddedFiles() bool {
+ return m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV2) HasExistingFiles() bool {
+ return m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV2) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+func fetchManifestEntries(m ManifestFile, fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ f, err := fs.Open(m.FilePath())
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+
+ dec, err := ocf.NewDecoder(f)
+ if err != nil {
+ return nil, err
+ }
+
+ metadata := dec.Metadata()
+ isVer1 := true
+ if string(metadata["format-version"]) == "2" {
+ isVer1 = false
+ }
+
+ results := make([]ManifestEntry, 0)
+ for dec.HasNext() {
+ var tmp ManifestEntry
+ if isVer1 {
+ tmp = &manifestEntryV1{}
+ } else {
+ tmp = &manifestEntryV2{}
+ }
+
+ if err := dec.Decode(tmp); err != nil {
+ return nil, err
+ }
+
+ if !discardDeleted || tmp.Status() != EntryStatusDELETED {
+ tmp.inheritSeqNum(m)
+ results = append(results, tmp)
+ }
+ }
+
+ return results, dec.Error()
+}
+
+// ManifestFile is the interface which covers both V1 and V2 manifest files.
+type ManifestFile interface {
+ // Version returns the version number of this manifest file.
+ // It should be 1 or 2.
+ Version() int
+ // FilePath is the location URI of this manifest file.
+ FilePath() string
+ // Length is the length in bytes of the manifest file.
+ Length() int64
+ // PartitionID is the ID of the partition spec used to write
+ // this manifest. It must be listed in the table metadata
+ // partition-specs.
+ PartitionID() int32
+ // ManifestContent is the type of files tracked by this manifest,
+ // either data or delete files. All v1 manifests track data files.
+ ManifestContent() ManifestContent
+ // SnapshotID is the ID of the snapshot where this manifest file
+ // was added.
+ SnapshotID() int64
+ // AddedDataFiles returns the number of entries in the manifest that
+ // have the status of EntryStatusADDED.
+ AddedDataFiles() int32
+ // ExistingDataFiles returns the number of entries in the manifest
+ // which have the status of EntryStatusEXISTING.
+ ExistingDataFiles() int32
+ // DeletedDataFiles returns the number of entries in the manifest
+ // which have the status of EntryStatusDELETED.
+ DeletedDataFiles() int32
+ // AddedRows returns the number of rows in all files of the manifest
+ // that have status EntryStatusADDED.
+ AddedRows() int64
+ // ExistingRows returns the number of rows in all files of the manifest
+ // which have status EntryStatusEXISTING.
+ ExistingRows() int64
+ // DeletedRows returns the number of rows in all files of the manifest
+ // which have status EntryStatusDELETED.
+ DeletedRows() int64
+ // SequenceNum returns the sequence number when this manifest was
+ // added to the table. Will be 0 for v1 manifest lists.
+ SequenceNum() int64
+ // MinSequenceNum is the minimum data sequence number of all live data
+ // or delete files in the manifest. Will be 0 for v1 manifest lists.
+ MinSequenceNum() int64
+ // Metadata returns implementation-specific key metadata for encryption
+ // if it exists in the manifest list.
+ Metadata() []byte
+ // PartitionList returns a list of field summaries for each partition
+ // field in the spec. Each field in the list corresponds to a field in
+ // the manifest file's partition spec.
+ PartitionList() []fieldSummary
+
+ // HasAddedFiles returns true if AddedDataFiles > 0 or if it was null.
+ HasAddedFiles() bool
+ // HasExistingFiles returns true if ExistingDataFiles > 0 or if it was null.
+ HasExistingFiles() bool
+ // FetchEntries reads the manifest list file to fetch the list of
+ // manifest entries using the provided file system IO interface.
+ // If discardDeleted is true, entries for files containing deleted rows
+ // will be skipped.
+ FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error)
+}
+
+// ReadManifestList reads in an avro manifest list file and returns a slice
+// of manifest files or an error if one is encountered.
+func ReadManifestList(in io.Reader) ([]ManifestFile, error) {
+ dec, err := ocf.NewDecoder(in)
+ if err != nil {
+ return nil, err
+ }
+
+ out := make([]ManifestFile, 0)
+
+ for dec.HasNext() {
+ var file ManifestFile
+ if string(dec.Metadata()["format-version"]) == "2" {
+ file = &manifestFileV2{}
+ } else {
+ file = &manifestFileV1{}
+ }
+
+ if err := dec.Decode(file); err != nil {
+ return nil, err
+ }
+ out = append(out, file)
+ }
+
+ return out, dec.Error()
+}
+
+// ManifestEntryStatus defines constants for the entry status of
+// existing, added or deleted.
+type ManifestEntryStatus int8
+
+const (
+ EntryStatusEXISTING ManifestEntryStatus = 0
+ EntryStatusADDED ManifestEntryStatus = 1
+ EntryStatusDELETED ManifestEntryStatus = 2
+)
+
+// ManifestEntryContent defines constants for the type of file contents
+// in the file entries. Data, Position based deletes and equality based
+// deletes.
+type ManifestEntryContent int8
+
+const (
+ EntryContentData ManifestEntryContent = 0
+ EntryContentPosDeletes ManifestEntryContent = 1
+ EntryContentEqDeletes ManifestEntryContent = 2
+)
+
+// FileFormat defines constants for the format of data files.
+type FileFormat string
+
+const (
+ AvroFile FileFormat = "AVRO"
+ OrcFile FileFormat = "ORC"
+ ParquetFile FileFormat = "PARQUET"
+)
+
+type colMap[K, V any] struct {
+ Key K `avro:"key"`
+ Value V `avro:"value"`
+}
+
+func avroColMapToMap[K comparable, V any](c *[]colMap[K, V]) map[K]V {
+ if c == nil {
+ return nil
+ }
+
+ out := make(map[K]V)
+ for _, data := range *c {
+ out[data.Key] = data.Value
+ }
+ return out
+}
+
+type dataFile struct {
+ Content ManifestEntryContent `avro:"content"`
Review Comment:
this only exists for V2 but not for V1
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] nastra commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1326977977
##########
manifest.go:
##########
@@ -0,0 +1,655 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package iceberg
+
+import (
+ "io"
+ "sync"
+
+ iceio "github.com/apache/iceberg-go/io"
+
+ "github.com/hamba/avro/v2/ocf"
+)
+
+// ManifestContent indicates the type of data inside of the files
+// described by a manifest. This will indicate whether the data files
+// contain active data or deleted rows.
+type ManifestContent int32
+
+const (
+ ManifestContentData ManifestContent = 0
+ ManifestContentDeletes ManifestContent = 1
+)
+
+type fieldSummary struct {
+ ContainsNull bool `avro:"contains_null"`
+ ContainsNaN *bool `avro:"contains_nan"`
+ LowerBound *[]byte `avro:"lower_bound"`
+ UpperBound *[]byte `avro:"upper_bound"`
+}
+
+type manifestFileV1 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ AddedSnapshotID *int64 `avro:"added_snapshot_id"`
+ AddedFilesCount *int32 `avro:"added_data_files_count"`
+ ExistingFilesCount *int32 `avro:"existing_data_files_count"`
+ DeletedFilesCount *int32 `avro:"deleted_data_files_count"`
+ AddedRowsCount *int64 `avro:"added_rows_count"`
+ ExistingRowsCount *int64 `avro:"existing_rows_count"`
+ DeletedRowsCount *int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV1) Version() int { return 1 }
+func (m *manifestFileV1) FilePath() string { return m.Path }
+func (m *manifestFileV1) Length() int64 { return m.Len }
+func (m *manifestFileV1) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV1) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV1) SnapshotID() int64 {
+ if m.AddedSnapshotID == nil {
+ return 0
+ }
+ return *m.AddedSnapshotID
+}
+
+func (m *manifestFileV1) AddedDataFiles() int32 {
+ if m.AddedFilesCount == nil {
+ return 0
+ }
+ return *m.AddedFilesCount
+}
+
+func (m *manifestFileV1) ExistingDataFiles() int32 {
+ if m.ExistingFilesCount == nil {
+ return 0
+ }
+ return *m.ExistingFilesCount
+}
+
+func (m *manifestFileV1) DeletedDataFiles() int32 {
+ if m.DeletedFilesCount == nil {
+ return 0
+ }
+ return *m.DeletedFilesCount
+}
+
+func (m *manifestFileV1) AddedRows() int64 {
+ if m.AddedRowsCount == nil {
+ return 0
+ }
+ return *m.AddedRowsCount
+}
+
+func (m *manifestFileV1) ExistingRows() int64 {
+ if m.ExistingRowsCount == nil {
+ return 0
+ }
+ return *m.ExistingRowsCount
+}
+
+func (m *manifestFileV1) DeletedRows() int64 {
+ if m.DeletedRowsCount == nil {
+ return 0
+ }
+ return *m.DeletedRowsCount
+}
+
+func (m *manifestFileV1) HasAddedFiles() bool {
+ return m.AddedFilesCount == nil || *m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV1) HasExistingFiles() bool {
+ return m.ExistingFilesCount == nil || *m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV1) SequenceNum() int64 { return 0 }
+func (m *manifestFileV1) MinSequenceNum() int64 { return 0 }
+func (m *manifestFileV1) Metadata() []byte { return m.KeyMetadata }
+func (m *manifestFileV1) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV1) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+type manifestFileV2 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ SeqNumber int64 `avro:"sequence_number"`
+ MinSeqNumber int64 `avro:"min_sequence_number"`
+ AddedSnapshotID int64 `avro:"added_snapshot_id"`
+ AddedFilesCount int32 `avro:"added_files_count"`
+ ExistingFilesCount int32 `avro:"existing_files_count"`
+ DeletedFilesCount int32 `avro:"deleted_files_count"`
+ AddedRowsCount int64 `avro:"added_rows_count"`
+ ExistingRowsCount int64 `avro:"existing_rows_count"`
+ DeletedRowsCount int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV2) Version() int { return 2 }
+
+func (m *manifestFileV2) FilePath() string { return m.Path }
+func (m *manifestFileV2) Length() int64 { return m.Len }
+func (m *manifestFileV2) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV2) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV2) SnapshotID() int64 {
+ return m.AddedSnapshotID
+}
+
+func (m *manifestFileV2) AddedDataFiles() int32 {
+ return m.AddedFilesCount
+}
+
+func (m *manifestFileV2) ExistingDataFiles() int32 {
+ return m.ExistingFilesCount
+}
+
+func (m *manifestFileV2) DeletedDataFiles() int32 {
+ return m.DeletedFilesCount
+}
+
+func (m *manifestFileV2) AddedRows() int64 {
+ return m.AddedRowsCount
+}
+
+func (m *manifestFileV2) ExistingRows() int64 {
+ return m.ExistingRowsCount
+}
+
+func (m *manifestFileV2) DeletedRows() int64 {
+ return m.DeletedRowsCount
+}
+
+func (m *manifestFileV2) SequenceNum() int64 { return m.SeqNumber }
+func (m *manifestFileV2) MinSequenceNum() int64 { return m.MinSeqNumber }
+func (m *manifestFileV2) Metadata() []byte { return m.KeyMetadata }
+
+func (m *manifestFileV2) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV2) HasAddedFiles() bool {
+ return m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV2) HasExistingFiles() bool {
+ return m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV2) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+func fetchManifestEntries(m ManifestFile, fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ f, err := fs.Open(m.FilePath())
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+
+ dec, err := ocf.NewDecoder(f)
+ if err != nil {
+ return nil, err
+ }
+
+ metadata := dec.Metadata()
+ isVer1 := true
+ if string(metadata["format-version"]) == "2" {
+ isVer1 = false
+ }
+
+ results := make([]ManifestEntry, 0)
+ for dec.HasNext() {
+ var tmp ManifestEntry
+ if isVer1 {
+ tmp = &manifestEntryV1{}
+ } else {
+ tmp = &manifestEntryV2{}
+ }
+
+ if err := dec.Decode(tmp); err != nil {
+ return nil, err
+ }
+
+ if !discardDeleted || tmp.Status() != EntryStatusDELETED {
+ tmp.inheritSeqNum(m)
+ results = append(results, tmp)
+ }
+ }
+
+ return results, dec.Error()
+}
+
+// ManifestFile is the interface which covers both V1 and V2 manifest files.
+type ManifestFile interface {
+ // Version returns the version number of this manifest file.
+ // It should be 1 or 2.
+ Version() int
+ // FilePath is the location URI of this manifest file.
+ FilePath() string
+ // Length is the length in bytes of the manifest file.
+ Length() int64
+ // PartitionID is the ID of the partition spec used to write
+ // this manifest. It must be listed in the table metadata
+ // partition-specs.
+ PartitionID() int32
Review Comment:
I think `PartitionSpecID()` is slightly better
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] nastra commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1326972514
##########
internal/avro_schemas.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package internal
+
+import "github.com/hamba/avro/v2"
+
+const (
+ ManifestListV1Key = "manifest-list-v1"
+ ManifestListV2Key = "manifest-list-v2"
+ ManifestEntryV1Key = "manifest-entry-v1"
+ ManifestEntryV2Key = "manifest-entry-v2"
+)
+
+var (
+ AvroSchemaCache avro.SchemaCache
+)
+
+func init() {
+ AvroSchemaCache.Add(ManifestListV1Key, avro.MustParse(`{
Review Comment:
this looks correct
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] nastra commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1326981591
##########
manifest.go:
##########
@@ -0,0 +1,655 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package iceberg
+
+import (
+ "io"
+ "sync"
+
+ iceio "github.com/apache/iceberg-go/io"
+
+ "github.com/hamba/avro/v2/ocf"
+)
+
+// ManifestContent indicates the type of data inside of the files
+// described by a manifest. This will indicate whether the data files
+// contain active data or deleted rows.
+type ManifestContent int32
+
+const (
+ ManifestContentData ManifestContent = 0
+ ManifestContentDeletes ManifestContent = 1
+)
+
+type fieldSummary struct {
+ ContainsNull bool `avro:"contains_null"`
+ ContainsNaN *bool `avro:"contains_nan"`
+ LowerBound *[]byte `avro:"lower_bound"`
+ UpperBound *[]byte `avro:"upper_bound"`
+}
+
+type manifestFileV1 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ AddedSnapshotID *int64 `avro:"added_snapshot_id"`
+ AddedFilesCount *int32 `avro:"added_data_files_count"`
+ ExistingFilesCount *int32 `avro:"existing_data_files_count"`
+ DeletedFilesCount *int32 `avro:"deleted_data_files_count"`
+ AddedRowsCount *int64 `avro:"added_rows_count"`
+ ExistingRowsCount *int64 `avro:"existing_rows_count"`
+ DeletedRowsCount *int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV1) Version() int { return 1 }
+func (m *manifestFileV1) FilePath() string { return m.Path }
+func (m *manifestFileV1) Length() int64 { return m.Len }
+func (m *manifestFileV1) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV1) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV1) SnapshotID() int64 {
+ if m.AddedSnapshotID == nil {
+ return 0
+ }
+ return *m.AddedSnapshotID
+}
+
+func (m *manifestFileV1) AddedDataFiles() int32 {
+ if m.AddedFilesCount == nil {
+ return 0
+ }
+ return *m.AddedFilesCount
+}
+
+func (m *manifestFileV1) ExistingDataFiles() int32 {
+ if m.ExistingFilesCount == nil {
+ return 0
+ }
+ return *m.ExistingFilesCount
+}
+
+func (m *manifestFileV1) DeletedDataFiles() int32 {
+ if m.DeletedFilesCount == nil {
+ return 0
+ }
+ return *m.DeletedFilesCount
+}
+
+func (m *manifestFileV1) AddedRows() int64 {
+ if m.AddedRowsCount == nil {
+ return 0
+ }
+ return *m.AddedRowsCount
+}
+
+func (m *manifestFileV1) ExistingRows() int64 {
+ if m.ExistingRowsCount == nil {
+ return 0
+ }
+ return *m.ExistingRowsCount
+}
+
+func (m *manifestFileV1) DeletedRows() int64 {
+ if m.DeletedRowsCount == nil {
+ return 0
+ }
+ return *m.DeletedRowsCount
+}
+
+func (m *manifestFileV1) HasAddedFiles() bool {
+ return m.AddedFilesCount == nil || *m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV1) HasExistingFiles() bool {
+ return m.ExistingFilesCount == nil || *m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV1) SequenceNum() int64 { return 0 }
+func (m *manifestFileV1) MinSequenceNum() int64 { return 0 }
+func (m *manifestFileV1) Metadata() []byte { return m.KeyMetadata }
+func (m *manifestFileV1) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV1) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+type manifestFileV2 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ SeqNumber int64 `avro:"sequence_number"`
+ MinSeqNumber int64 `avro:"min_sequence_number"`
+ AddedSnapshotID int64 `avro:"added_snapshot_id"`
+ AddedFilesCount int32 `avro:"added_files_count"`
+ ExistingFilesCount int32 `avro:"existing_files_count"`
+ DeletedFilesCount int32 `avro:"deleted_files_count"`
+ AddedRowsCount int64 `avro:"added_rows_count"`
+ ExistingRowsCount int64 `avro:"existing_rows_count"`
+ DeletedRowsCount int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV2) Version() int { return 2 }
+
+func (m *manifestFileV2) FilePath() string { return m.Path }
+func (m *manifestFileV2) Length() int64 { return m.Len }
+func (m *manifestFileV2) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV2) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV2) SnapshotID() int64 {
+ return m.AddedSnapshotID
+}
+
+func (m *manifestFileV2) AddedDataFiles() int32 {
+ return m.AddedFilesCount
+}
+
+func (m *manifestFileV2) ExistingDataFiles() int32 {
+ return m.ExistingFilesCount
+}
+
+func (m *manifestFileV2) DeletedDataFiles() int32 {
+ return m.DeletedFilesCount
+}
+
+func (m *manifestFileV2) AddedRows() int64 {
+ return m.AddedRowsCount
+}
+
+func (m *manifestFileV2) ExistingRows() int64 {
+ return m.ExistingRowsCount
+}
+
+func (m *manifestFileV2) DeletedRows() int64 {
+ return m.DeletedRowsCount
+}
+
+func (m *manifestFileV2) SequenceNum() int64 { return m.SeqNumber }
+func (m *manifestFileV2) MinSequenceNum() int64 { return m.MinSeqNumber }
+func (m *manifestFileV2) Metadata() []byte { return m.KeyMetadata }
+
+func (m *manifestFileV2) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV2) HasAddedFiles() bool {
+ return m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV2) HasExistingFiles() bool {
+ return m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV2) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+func fetchManifestEntries(m ManifestFile, fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ f, err := fs.Open(m.FilePath())
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+
+ dec, err := ocf.NewDecoder(f)
+ if err != nil {
+ return nil, err
+ }
+
+ metadata := dec.Metadata()
+ isVer1 := true
+ if string(metadata["format-version"]) == "2" {
+ isVer1 = false
+ }
+
+ results := make([]ManifestEntry, 0)
+ for dec.HasNext() {
+ var tmp ManifestEntry
+ if isVer1 {
+ tmp = &manifestEntryV1{}
+ } else {
+ tmp = &manifestEntryV2{}
+ }
+
+ if err := dec.Decode(tmp); err != nil {
+ return nil, err
+ }
+
+ if !discardDeleted || tmp.Status() != EntryStatusDELETED {
+ tmp.inheritSeqNum(m)
+ results = append(results, tmp)
+ }
+ }
+
+ return results, dec.Error()
+}
+
+// ManifestFile is the interface which covers both V1 and V2 manifest files.
+type ManifestFile interface {
+ // Version returns the version number of this manifest file.
+ // It should be 1 or 2.
+ Version() int
+ // FilePath is the location URI of this manifest file.
+ FilePath() string
+ // Length is the length in bytes of the manifest file.
+ Length() int64
+ // PartitionID is the ID of the partition spec used to write
+ // this manifest. It must be listed in the table metadata
+ // partition-specs.
+ PartitionID() int32
+ // ManifestContent is the type of files tracked by this manifest,
+ // either data or delete files. All v1 manifests track data files.
+ ManifestContent() ManifestContent
+ // SnapshotID is the ID of the snapshot where this manifest file
+ // was added.
+ SnapshotID() int64
+ // AddedDataFiles returns the number of entries in the manifest that
+ // have the status of EntryStatusADDED.
+ AddedDataFiles() int32
+ // ExistingDataFiles returns the number of entries in the manifest
+ // which have the status of EntryStatusEXISTING.
+ ExistingDataFiles() int32
+ // DeletedDataFiles returns the number of entries in the manifest
+ // which have the status of EntryStatusDELETED.
+ DeletedDataFiles() int32
+ // AddedRows returns the number of rows in all files of the manifest
+ // that have status EntryStatusADDED.
+ AddedRows() int64
+ // ExistingRows returns the number of rows in all files of the manifest
+ // which have status EntryStatusEXISTING.
+ ExistingRows() int64
+ // DeletedRows returns the number of rows in all files of the manifest
+ // which have status EntryStatusDELETED.
+ DeletedRows() int64
+ // SequenceNum returns the sequence number when this manifest was
+ // added to the table. Will be 0 for v1 manifest lists.
+ SequenceNum() int64
+ // MinSequenceNum is the minimum data sequence number of all live data
+ // or delete files in the manifest. Will be 0 for v1 manifest lists.
+ MinSequenceNum() int64
+ // Metadata returns implementation-specific key metadata for encryption
+ // if it exists in the manifest list.
+ Metadata() []byte
Review Comment:
I'd rather name this `KeyMetadata()` to have a clear distinction later between this and table metadata
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] rdblue commented on pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#issuecomment-1710410048
@zeroshade, sorry for the delay! I'll find some time for reviews over here.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] nastra commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1326995463
##########
manifest.go:
##########
@@ -0,0 +1,655 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package iceberg
+
+import (
+ "io"
+ "sync"
+
+ iceio "github.com/apache/iceberg-go/io"
+
+ "github.com/hamba/avro/v2/ocf"
+)
+
+// ManifestContent indicates the type of data inside of the files
+// described by a manifest. This will indicate whether the data files
+// contain active data or deleted rows.
+type ManifestContent int32
+
+const (
+ ManifestContentData ManifestContent = 0
+ ManifestContentDeletes ManifestContent = 1
+)
+
+type fieldSummary struct {
+ ContainsNull bool `avro:"contains_null"`
+ ContainsNaN *bool `avro:"contains_nan"`
+ LowerBound *[]byte `avro:"lower_bound"`
+ UpperBound *[]byte `avro:"upper_bound"`
+}
+
+type manifestFileV1 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ AddedSnapshotID *int64 `avro:"added_snapshot_id"`
+ AddedFilesCount *int32 `avro:"added_data_files_count"`
+ ExistingFilesCount *int32 `avro:"existing_data_files_count"`
+ DeletedFilesCount *int32 `avro:"deleted_data_files_count"`
+ AddedRowsCount *int64 `avro:"added_rows_count"`
+ ExistingRowsCount *int64 `avro:"existing_rows_count"`
+ DeletedRowsCount *int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV1) Version() int { return 1 }
+func (m *manifestFileV1) FilePath() string { return m.Path }
+func (m *manifestFileV1) Length() int64 { return m.Len }
+func (m *manifestFileV1) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV1) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV1) SnapshotID() int64 {
+ if m.AddedSnapshotID == nil {
+ return 0
+ }
+ return *m.AddedSnapshotID
+}
+
+func (m *manifestFileV1) AddedDataFiles() int32 {
+ if m.AddedFilesCount == nil {
+ return 0
+ }
+ return *m.AddedFilesCount
+}
+
+func (m *manifestFileV1) ExistingDataFiles() int32 {
+ if m.ExistingFilesCount == nil {
+ return 0
+ }
+ return *m.ExistingFilesCount
+}
+
+func (m *manifestFileV1) DeletedDataFiles() int32 {
+ if m.DeletedFilesCount == nil {
+ return 0
+ }
+ return *m.DeletedFilesCount
+}
+
+func (m *manifestFileV1) AddedRows() int64 {
+ if m.AddedRowsCount == nil {
+ return 0
+ }
+ return *m.AddedRowsCount
+}
+
+func (m *manifestFileV1) ExistingRows() int64 {
+ if m.ExistingRowsCount == nil {
+ return 0
+ }
+ return *m.ExistingRowsCount
+}
+
+func (m *manifestFileV1) DeletedRows() int64 {
+ if m.DeletedRowsCount == nil {
+ return 0
+ }
+ return *m.DeletedRowsCount
+}
+
+func (m *manifestFileV1) HasAddedFiles() bool {
+ return m.AddedFilesCount == nil || *m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV1) HasExistingFiles() bool {
+ return m.ExistingFilesCount == nil || *m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV1) SequenceNum() int64 { return 0 }
+func (m *manifestFileV1) MinSequenceNum() int64 { return 0 }
+func (m *manifestFileV1) Metadata() []byte { return m.KeyMetadata }
+func (m *manifestFileV1) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV1) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+type manifestFileV2 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ SeqNumber int64 `avro:"sequence_number"`
+ MinSeqNumber int64 `avro:"min_sequence_number"`
+ AddedSnapshotID int64 `avro:"added_snapshot_id"`
+ AddedFilesCount int32 `avro:"added_files_count"`
+ ExistingFilesCount int32 `avro:"existing_files_count"`
+ DeletedFilesCount int32 `avro:"deleted_files_count"`
+ AddedRowsCount int64 `avro:"added_rows_count"`
+ ExistingRowsCount int64 `avro:"existing_rows_count"`
+ DeletedRowsCount int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV2) Version() int { return 2 }
+
+func (m *manifestFileV2) FilePath() string { return m.Path }
+func (m *manifestFileV2) Length() int64 { return m.Len }
+func (m *manifestFileV2) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV2) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV2) SnapshotID() int64 {
+ return m.AddedSnapshotID
+}
+
+func (m *manifestFileV2) AddedDataFiles() int32 {
+ return m.AddedFilesCount
+}
+
+func (m *manifestFileV2) ExistingDataFiles() int32 {
+ return m.ExistingFilesCount
+}
+
+func (m *manifestFileV2) DeletedDataFiles() int32 {
+ return m.DeletedFilesCount
+}
+
+func (m *manifestFileV2) AddedRows() int64 {
+ return m.AddedRowsCount
+}
+
+func (m *manifestFileV2) ExistingRows() int64 {
+ return m.ExistingRowsCount
+}
+
+func (m *manifestFileV2) DeletedRows() int64 {
+ return m.DeletedRowsCount
+}
+
+func (m *manifestFileV2) SequenceNum() int64 { return m.SeqNumber }
+func (m *manifestFileV2) MinSequenceNum() int64 { return m.MinSeqNumber }
+func (m *manifestFileV2) Metadata() []byte { return m.KeyMetadata }
+
+func (m *manifestFileV2) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV2) HasAddedFiles() bool {
+ return m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV2) HasExistingFiles() bool {
+ return m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV2) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+func fetchManifestEntries(m ManifestFile, fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ f, err := fs.Open(m.FilePath())
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+
+ dec, err := ocf.NewDecoder(f)
+ if err != nil {
+ return nil, err
+ }
+
+ metadata := dec.Metadata()
+ isVer1 := true
+ if string(metadata["format-version"]) == "2" {
+ isVer1 = false
+ }
+
+ results := make([]ManifestEntry, 0)
+ for dec.HasNext() {
+ var tmp ManifestEntry
+ if isVer1 {
+ tmp = &manifestEntryV1{}
+ } else {
+ tmp = &manifestEntryV2{}
+ }
+
+ if err := dec.Decode(tmp); err != nil {
+ return nil, err
+ }
+
+ if !discardDeleted || tmp.Status() != EntryStatusDELETED {
+ tmp.inheritSeqNum(m)
+ results = append(results, tmp)
+ }
+ }
+
+ return results, dec.Error()
+}
+
+// ManifestFile is the interface which covers both V1 and V2 manifest files.
+type ManifestFile interface {
+ // Version returns the version number of this manifest file.
+ // It should be 1 or 2.
+ Version() int
+ // FilePath is the location URI of this manifest file.
+ FilePath() string
+ // Length is the length in bytes of the manifest file.
+ Length() int64
+ // PartitionID is the ID of the partition spec used to write
+ // this manifest. It must be listed in the table metadata
+ // partition-specs.
+ PartitionID() int32
+ // ManifestContent is the type of files tracked by this manifest,
+ // either data or delete files. All v1 manifests track data files.
+ ManifestContent() ManifestContent
+ // SnapshotID is the ID of the snapshot where this manifest file
+ // was added.
+ SnapshotID() int64
+ // AddedDataFiles returns the number of entries in the manifest that
+ // have the status of EntryStatusADDED.
+ AddedDataFiles() int32
+ // ExistingDataFiles returns the number of entries in the manifest
+ // which have the status of EntryStatusEXISTING.
+ ExistingDataFiles() int32
+ // DeletedDataFiles returns the number of entries in the manifest
+ // which have the status of EntryStatusDELETED.
+ DeletedDataFiles() int32
+ // AddedRows returns the number of rows in all files of the manifest
+ // that have status EntryStatusADDED.
+ AddedRows() int64
+ // ExistingRows returns the number of rows in all files of the manifest
+ // which have status EntryStatusEXISTING.
+ ExistingRows() int64
+ // DeletedRows returns the number of rows in all files of the manifest
+ // which have status EntryStatusDELETED.
+ DeletedRows() int64
+ // SequenceNum returns the sequence number when this manifest was
+ // added to the table. Will be 0 for v1 manifest lists.
+ SequenceNum() int64
+ // MinSequenceNum is the minimum data sequence number of all live data
+ // or delete files in the manifest. Will be 0 for v1 manifest lists.
+ MinSequenceNum() int64
+ // Metadata returns implementation-specific key metadata for encryption
+ // if it exists in the manifest list.
+ Metadata() []byte
+ // PartitionList returns a list of field summaries for each partition
+ // field in the spec. Each field in the list corresponds to a field in
+ // the manifest file's partition spec.
+ PartitionList() []fieldSummary
+
+ // HasAddedFiles returns true if AddedDataFiles > 0 or if it was null.
+ HasAddedFiles() bool
+ // HasExistingFiles returns true if ExistingDataFiles > 0 or if it was null.
+ HasExistingFiles() bool
+ // FetchEntries reads the manifest list file to fetch the list of
+ // manifest entries using the provided file system IO interface.
+ // If discardDeleted is true, entries for files containing deleted rows
+ // will be skipped.
+ FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error)
+}
+
+// ReadManifestList reads in an avro manifest list file and returns a slice
+// of manifest files or an error if one is encountered.
+func ReadManifestList(in io.Reader) ([]ManifestFile, error) {
+ dec, err := ocf.NewDecoder(in)
+ if err != nil {
+ return nil, err
+ }
+
+ out := make([]ManifestFile, 0)
+
+ for dec.HasNext() {
+ var file ManifestFile
+ if string(dec.Metadata()["format-version"]) == "2" {
+ file = &manifestFileV2{}
+ } else {
+ file = &manifestFileV1{}
+ }
+
+ if err := dec.Decode(file); err != nil {
+ return nil, err
+ }
+ out = append(out, file)
+ }
+
+ return out, dec.Error()
+}
+
+// ManifestEntryStatus defines constants for the entry status of
+// existing, added or deleted.
+type ManifestEntryStatus int8
+
+const (
+ EntryStatusEXISTING ManifestEntryStatus = 0
+ EntryStatusADDED ManifestEntryStatus = 1
+ EntryStatusDELETED ManifestEntryStatus = 2
+)
+
+// ManifestEntryContent defines constants for the type of file contents
+// in the file entries. Data, Position based deletes and equality based
+// deletes.
+type ManifestEntryContent int8
+
+const (
+ EntryContentData ManifestEntryContent = 0
+ EntryContentPosDeletes ManifestEntryContent = 1
+ EntryContentEqDeletes ManifestEntryContent = 2
+)
+
+// FileFormat defines constants for the format of data files.
+type FileFormat string
+
+const (
+ AvroFile FileFormat = "AVRO"
+ OrcFile FileFormat = "ORC"
+ ParquetFile FileFormat = "PARQUET"
+)
+
+type colMap[K, V any] struct {
+ Key K `avro:"key"`
+ Value V `avro:"value"`
+}
+
+func avroColMapToMap[K comparable, V any](c *[]colMap[K, V]) map[K]V {
+ if c == nil {
+ return nil
+ }
+
+ out := make(map[K]V)
+ for _, data := range *c {
+ out[data.Key] = data.Value
+ }
+ return out
+}
+
+type dataFile struct {
+ Content ManifestEntryContent `avro:"content"`
+ Path string `avro:"file_path"`
+ Format FileFormat `avro:"file_format"`
+ PartitionData map[string]any `avro:"partition"`
+ RecordCount int64 `avro:"record_count"`
+ FileSize int64 `avro:"file_size_in_bytes"`
+ BlockSizeInBytes int64 `avro:"block_size_in_bytes"`
+ ColSizes *[]colMap[int, int64] `avro:"column_sizes"`
+ ValCounts *[]colMap[int, int64] `avro:"value_counts"`
+ NullCounts *[]colMap[int, int64] `avro:"null_value_counts"`
+ NaNCounts *[]colMap[int, int64] `avro:"nan_value_counts"`
+ DistinctCounts *[]colMap[int, int64] `avro:"distinct_counts"`
+ LowerBounds *[]colMap[int, []byte] `avro:"lower_bounds"`
+ UpperBounds *[]colMap[int, []byte] `avro:"upper_bounds"`
+ KeyMetadata *[]byte `avro:"key_metadata"`
+ SplitOffsets *[]int64 `avro:"split_offsets"`
+ EqualityIDs *[]int `avro:"equality_ids"`
Review Comment:
this only exists for V2 but not for V1
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] nastra commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1325910141
##########
io/io.go:
##########
@@ -0,0 +1,327 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package io
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "io/fs"
+ "net/http"
+ "net/url"
+ "os"
+ "strings"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
+ "github.com/aws/aws-sdk-go-v2/config"
+ "github.com/aws/aws-sdk-go-v2/credentials"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+ "github.com/wolfeidau/s3iofs"
+)
+
+const (
+ S3RegionName = "s3.region-name"
+ S3SessionToken = "s3.session-token"
+ S3SecretAccessKey = "s3.secret-access-key"
+ S3AccessKeyID = "s3.access-key-id"
+ S3EndpointURL = "s3.region"
+ S3ProxyURI = "s3.proxy-uri"
+)
+
+// IO is an interface to a hierarchical file system.
+//
+// The IO interface is the minimum implementation required for a file
+// system to utilize an iceberg table. A file system may implement
+// additional interfaces, such as ReadFileIO, to provide additional or
+// optimized functionality.
+type IO interface {
+ // Open opens the named file.
+ //
+ // When Open returns an error, it should be of type *PathError
+ // with the Op field set to "open", the Path field set to name,
+ // and the Err field describing the problem.
+ //
+ // Open should reject attempts to open names that do not satisfy
+ // fs.ValidPath(name), returning a *PathError with Err set to
+ // ErrInvalid or ErrNotExist.
+ Open(name string) (File, error)
+
+ // Remove removes the named file or (empty) directory.
+ //
+ // If there is an error, it will be of type *PathError.
+ Remove(name string) error
+}
+
+// ReadFileIO is the interface implemented by a file system that
+// provides an optimized implementation of ReadFile.
+type ReadFileIO interface {
+ IO
+
+ // ReadFile reads the named file and returns its contents.
+ // A successful call returns a nil error, not io.EOF.
+ // (Because ReadFile reads the whole file, the expected EOF
+ // from the final Read is not treated as an error to be reported.)
+ //
+ // The caller is permitted to modify the returned byte slice.
+ // This method should return a copy of the underlying data.
+ ReadFile(name string) ([]byte, error)
+}
+
+// A File provides access to a single file. The File interface is the
+// minimum implementation required for Iceberg to interact with a file.
+// Directory files should also implement
+type File interface {
+ fs.File
+ io.ReadSeekCloser
+ io.ReaderAt
+}
+
+// A ReadDirFile is a directory file whose entries can be read with the
+// ReadDir method. Every directory file should implement this interface.
+// (It is permissible for any file to implement this interface, but
+// if so ReadDir should return an error for non-directories.)
+type ReadDirFile interface {
+ File
+
+ // ReadDir read the contents of the directory and returns a slice
+ // of up to n DirEntry values in directory order. Subsequent calls
+ // on the same file will yield further DirEntry values.
+ //
+ // If n > 0, ReadDir returns at most n DirEntry structures. In this
+ // case, if ReadDir returns an empty slice, it will return a non-nil
+ // error explaining why.
+ //
+ // At the end of a directory, the error is io.EOF. (ReadDir must return
+ // io.EOF itself, not an error wrapping io.EOF.)
+ //
+ // If n <= 0, ReadDir returns all the DirEntry values from the directory
+ // in a single slice. In this case, if ReadDir succeeds (reads all the way
+ // to the end of the directory), it returns the slice and a nil error.
+ // If it encounters an error before the end of the directory, ReadDir
+ // returns the DirEntry list read until that point and a non-nil error.
+ ReadDir(n int) ([]fs.DirEntry, error)
+}
+
+// FS wraps an io/fs.FS as an IO interface.
+func FS(fsys fs.FS) IO {
+ if _, ok := fsys.(fs.ReadFileFS); ok {
+ return readFileFS{ioFS{fsys, nil}}
+ }
+ return ioFS{fsys, nil}
+}
+
+// FSPreProcName wraps an io/fs.FS like FS, only if fn is non-nil then
+// it is called to preprocess any filenames before they are passed to
+// the underlying fsys.
+func FSPreProcName(fsys fs.FS, fn func(string) string) IO {
+ if _, ok := fsys.(fs.ReadFileFS); ok {
+ return readFileFS{ioFS{fsys, fn}}
+ }
+ return ioFS{fsys, fn}
+}
+
+type readFileFS struct {
+ ioFS
+}
+
+func (r readFileFS) ReadFile(name string) ([]byte, error) {
+ if r.preProcessName != nil {
+ name = r.preProcessName(name)
+ }
+
+ rfs, ok := r.fsys.(fs.ReadFileFS)
+ if !ok {
+ return nil, errMissingReadFile
+ }
+ return rfs.ReadFile(name)
+}
+
+type ioFS struct {
+ fsys fs.FS
+
+ preProcessName func(string) string
+}
+
+func (f ioFS) Open(name string) (File, error) {
+ if f.preProcessName != nil {
+ name = f.preProcessName(name)
+ }
+
+ if name == "/" {
+ name = "."
+ } else {
+ name = strings.TrimPrefix(name, "/")
+ }
+ file, err := f.fsys.Open(name)
+ if err != nil {
+ return nil, err
+ }
+
+ return ioFile{file}, nil
+}
+
+func (f ioFS) Remove(name string) error {
+ r, ok := f.fsys.(interface{ Remove(name string) error })
+ if !ok {
+ return errMissingRemove
+ }
+ return r.Remove(name)
+}
+
+var (
+ errMissingReadDir = errors.New("fs.File directory missing ReadDir method")
+ errMissingSeek = errors.New("fs.File missing Seek method")
+ errMissingReadAt = errors.New("fs.File missing ReadAt")
+ errMissingRemove = errors.New("fs.FS missing Remove method")
+ errMissingReadFile = errors.New("fs.FS missing ReadFile method")
+)
+
+type ioFile struct {
+ file fs.File
+}
+
+func (f ioFile) Close() error { return f.file.Close() }
+func (f ioFile) Read(b []byte) (int, error) { return f.file.Read(b) }
+func (f ioFile) Stat() (fs.FileInfo, error) { return f.file.Stat() }
+func (f ioFile) Seek(offset int64, whence int) (int64, error) {
+ s, ok := f.file.(io.Seeker)
+ if !ok {
+ return 0, errMissingSeek
+ }
+ return s.Seek(offset, whence)
+}
+
+func (f ioFile) ReadAt(p []byte, off int64) (n int, err error) {
+ r, ok := f.file.(io.ReaderAt)
+ if !ok {
+ return 0, errMissingReadAt
+ }
+ return r.ReadAt(p, off)
+}
+
+func (f ioFile) ReadDir(count int) ([]fs.DirEntry, error) {
+ d, ok := f.file.(fs.ReadDirFile)
+ if !ok {
+ return nil, errMissingReadDir
+ }
+
+ return d.ReadDir(count)
+}
+
+func inferFileIOFromSchema(path string, props map[string]string) (IO, error) {
+ parsed, err := url.Parse(path)
+ if err != nil {
+ return nil, err
+ }
+
+ switch parsed.Scheme {
+ case "s3", "s3a", "s3n":
+ opts := []func(*config.LoadOptions) error{}
+ endpoint, ok := props[S3EndpointURL]
+ if !ok {
+ endpoint = os.Getenv("AWS_S3_ENDPOINT")
+ }
+
+ if endpoint != "" {
Review Comment:
I would rather move all of this s3-related things to s3.go or something. Otherwise this code here will get quite difficult to read later when e.g. gcs or azure support will be added
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] nastra commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1325904401
##########
io/io.go:
##########
@@ -0,0 +1,327 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package io
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "io/fs"
+ "net/http"
+ "net/url"
+ "os"
+ "strings"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
+ "github.com/aws/aws-sdk-go-v2/config"
+ "github.com/aws/aws-sdk-go-v2/credentials"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+ "github.com/wolfeidau/s3iofs"
+)
+
+const (
+ S3RegionName = "s3.region-name"
Review Comment:
```suggestion
S3Region = "s3.region"
```
to be consistent with pyiceberg
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] nastra commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1327004629
##########
manifest.go:
##########
@@ -0,0 +1,655 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package iceberg
+
+import (
+ "io"
+ "sync"
+
+ iceio "github.com/apache/iceberg-go/io"
+
+ "github.com/hamba/avro/v2/ocf"
+)
+
+// ManifestContent indicates the type of data inside of the files
+// described by a manifest. This will indicate whether the data files
+// contain active data or deleted rows.
+type ManifestContent int32
+
+const (
+ ManifestContentData ManifestContent = 0
+ ManifestContentDeletes ManifestContent = 1
+)
+
+type fieldSummary struct {
+ ContainsNull bool `avro:"contains_null"`
+ ContainsNaN *bool `avro:"contains_nan"`
+ LowerBound *[]byte `avro:"lower_bound"`
+ UpperBound *[]byte `avro:"upper_bound"`
+}
+
+type manifestFileV1 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ AddedSnapshotID *int64 `avro:"added_snapshot_id"`
+ AddedFilesCount *int32 `avro:"added_data_files_count"`
+ ExistingFilesCount *int32 `avro:"existing_data_files_count"`
+ DeletedFilesCount *int32 `avro:"deleted_data_files_count"`
+ AddedRowsCount *int64 `avro:"added_rows_count"`
+ ExistingRowsCount *int64 `avro:"existing_rows_count"`
+ DeletedRowsCount *int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV1) Version() int { return 1 }
+func (m *manifestFileV1) FilePath() string { return m.Path }
+func (m *manifestFileV1) Length() int64 { return m.Len }
+func (m *manifestFileV1) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV1) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV1) SnapshotID() int64 {
+ if m.AddedSnapshotID == nil {
+ return 0
+ }
+ return *m.AddedSnapshotID
+}
+
+func (m *manifestFileV1) AddedDataFiles() int32 {
+ if m.AddedFilesCount == nil {
+ return 0
+ }
+ return *m.AddedFilesCount
+}
+
+func (m *manifestFileV1) ExistingDataFiles() int32 {
+ if m.ExistingFilesCount == nil {
+ return 0
+ }
+ return *m.ExistingFilesCount
+}
+
+func (m *manifestFileV1) DeletedDataFiles() int32 {
+ if m.DeletedFilesCount == nil {
+ return 0
+ }
+ return *m.DeletedFilesCount
+}
+
+func (m *manifestFileV1) AddedRows() int64 {
+ if m.AddedRowsCount == nil {
+ return 0
+ }
+ return *m.AddedRowsCount
+}
+
+func (m *manifestFileV1) ExistingRows() int64 {
+ if m.ExistingRowsCount == nil {
+ return 0
+ }
+ return *m.ExistingRowsCount
+}
+
+func (m *manifestFileV1) DeletedRows() int64 {
+ if m.DeletedRowsCount == nil {
+ return 0
+ }
+ return *m.DeletedRowsCount
+}
+
+func (m *manifestFileV1) HasAddedFiles() bool {
+ return m.AddedFilesCount == nil || *m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV1) HasExistingFiles() bool {
+ return m.ExistingFilesCount == nil || *m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV1) SequenceNum() int64 { return 0 }
+func (m *manifestFileV1) MinSequenceNum() int64 { return 0 }
+func (m *manifestFileV1) Metadata() []byte { return m.KeyMetadata }
+func (m *manifestFileV1) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV1) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+type manifestFileV2 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ SeqNumber int64 `avro:"sequence_number"`
+ MinSeqNumber int64 `avro:"min_sequence_number"`
+ AddedSnapshotID int64 `avro:"added_snapshot_id"`
+ AddedFilesCount int32 `avro:"added_files_count"`
+ ExistingFilesCount int32 `avro:"existing_files_count"`
+ DeletedFilesCount int32 `avro:"deleted_files_count"`
+ AddedRowsCount int64 `avro:"added_rows_count"`
+ ExistingRowsCount int64 `avro:"existing_rows_count"`
+ DeletedRowsCount int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV2) Version() int { return 2 }
+
+func (m *manifestFileV2) FilePath() string { return m.Path }
+func (m *manifestFileV2) Length() int64 { return m.Len }
+func (m *manifestFileV2) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV2) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV2) SnapshotID() int64 {
+ return m.AddedSnapshotID
+}
+
+func (m *manifestFileV2) AddedDataFiles() int32 {
+ return m.AddedFilesCount
+}
+
+func (m *manifestFileV2) ExistingDataFiles() int32 {
+ return m.ExistingFilesCount
+}
+
+func (m *manifestFileV2) DeletedDataFiles() int32 {
+ return m.DeletedFilesCount
+}
+
+func (m *manifestFileV2) AddedRows() int64 {
+ return m.AddedRowsCount
+}
+
+func (m *manifestFileV2) ExistingRows() int64 {
+ return m.ExistingRowsCount
+}
+
+func (m *manifestFileV2) DeletedRows() int64 {
+ return m.DeletedRowsCount
+}
+
+func (m *manifestFileV2) SequenceNum() int64 { return m.SeqNumber }
+func (m *manifestFileV2) MinSequenceNum() int64 { return m.MinSeqNumber }
+func (m *manifestFileV2) Metadata() []byte { return m.KeyMetadata }
+
+func (m *manifestFileV2) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV2) HasAddedFiles() bool {
+ return m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV2) HasExistingFiles() bool {
+ return m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV2) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+func fetchManifestEntries(m ManifestFile, fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ f, err := fs.Open(m.FilePath())
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+
+ dec, err := ocf.NewDecoder(f)
+ if err != nil {
+ return nil, err
+ }
+
+ metadata := dec.Metadata()
+ isVer1 := true
+ if string(metadata["format-version"]) == "2" {
+ isVer1 = false
+ }
+
+ results := make([]ManifestEntry, 0)
+ for dec.HasNext() {
+ var tmp ManifestEntry
+ if isVer1 {
+ tmp = &manifestEntryV1{}
+ } else {
+ tmp = &manifestEntryV2{}
+ }
+
+ if err := dec.Decode(tmp); err != nil {
+ return nil, err
+ }
+
+ if !discardDeleted || tmp.Status() != EntryStatusDELETED {
+ tmp.inheritSeqNum(m)
+ results = append(results, tmp)
+ }
+ }
+
+ return results, dec.Error()
+}
+
+// ManifestFile is the interface which covers both V1 and V2 manifest files.
+type ManifestFile interface {
+ // Version returns the version number of this manifest file.
+ // It should be 1 or 2.
+ Version() int
+ // FilePath is the location URI of this manifest file.
+ FilePath() string
+ // Length is the length in bytes of the manifest file.
+ Length() int64
+ // PartitionID is the ID of the partition spec used to write
+ // this manifest. It must be listed in the table metadata
+ // partition-specs.
+ PartitionID() int32
+ // ManifestContent is the type of files tracked by this manifest,
+ // either data or delete files. All v1 manifests track data files.
+ ManifestContent() ManifestContent
+ // SnapshotID is the ID of the snapshot where this manifest file
+ // was added.
+ SnapshotID() int64
+ // AddedDataFiles returns the number of entries in the manifest that
+ // have the status of EntryStatusADDED.
+ AddedDataFiles() int32
+ // ExistingDataFiles returns the number of entries in the manifest
+ // which have the status of EntryStatusEXISTING.
+ ExistingDataFiles() int32
+ // DeletedDataFiles returns the number of entries in the manifest
+ // which have the status of EntryStatusDELETED.
+ DeletedDataFiles() int32
+ // AddedRows returns the number of rows in all files of the manifest
+ // that have status EntryStatusADDED.
+ AddedRows() int64
+ // ExistingRows returns the number of rows in all files of the manifest
+ // which have status EntryStatusEXISTING.
+ ExistingRows() int64
+ // DeletedRows returns the number of rows in all files of the manifest
+ // which have status EntryStatusDELETED.
+ DeletedRows() int64
+ // SequenceNum returns the sequence number when this manifest was
+ // added to the table. Will be 0 for v1 manifest lists.
+ SequenceNum() int64
+ // MinSequenceNum is the minimum data sequence number of all live data
+ // or delete files in the manifest. Will be 0 for v1 manifest lists.
+ MinSequenceNum() int64
+ // Metadata returns implementation-specific key metadata for encryption
+ // if it exists in the manifest list.
+ Metadata() []byte
+ // PartitionList returns a list of field summaries for each partition
+ // field in the spec. Each field in the list corresponds to a field in
+ // the manifest file's partition spec.
+ PartitionList() []fieldSummary
+
+ // HasAddedFiles returns true if AddedDataFiles > 0 or if it was null.
+ HasAddedFiles() bool
+ // HasExistingFiles returns true if ExistingDataFiles > 0 or if it was null.
+ HasExistingFiles() bool
+ // FetchEntries reads the manifest list file to fetch the list of
+ // manifest entries using the provided file system IO interface.
+ // If discardDeleted is true, entries for files containing deleted rows
+ // will be skipped.
+ FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error)
+}
+
+// ReadManifestList reads in an avro manifest list file and returns a slice
+// of manifest files or an error if one is encountered.
+func ReadManifestList(in io.Reader) ([]ManifestFile, error) {
+ dec, err := ocf.NewDecoder(in)
+ if err != nil {
+ return nil, err
+ }
+
+ out := make([]ManifestFile, 0)
+
+ for dec.HasNext() {
+ var file ManifestFile
+ if string(dec.Metadata()["format-version"]) == "2" {
+ file = &manifestFileV2{}
+ } else {
+ file = &manifestFileV1{}
+ }
+
+ if err := dec.Decode(file); err != nil {
+ return nil, err
+ }
+ out = append(out, file)
+ }
+
+ return out, dec.Error()
+}
+
+// ManifestEntryStatus defines constants for the entry status of
+// existing, added or deleted.
+type ManifestEntryStatus int8
+
+const (
+ EntryStatusEXISTING ManifestEntryStatus = 0
+ EntryStatusADDED ManifestEntryStatus = 1
+ EntryStatusDELETED ManifestEntryStatus = 2
+)
+
+// ManifestEntryContent defines constants for the type of file contents
+// in the file entries. Data, Position based deletes and equality based
+// deletes.
+type ManifestEntryContent int8
+
+const (
+ EntryContentData ManifestEntryContent = 0
+ EntryContentPosDeletes ManifestEntryContent = 1
+ EntryContentEqDeletes ManifestEntryContent = 2
+)
+
+// FileFormat defines constants for the format of data files.
+type FileFormat string
+
+const (
+ AvroFile FileFormat = "AVRO"
+ OrcFile FileFormat = "ORC"
+ ParquetFile FileFormat = "PARQUET"
+)
+
+type colMap[K, V any] struct {
+ Key K `avro:"key"`
+ Value V `avro:"value"`
+}
+
+func avroColMapToMap[K comparable, V any](c *[]colMap[K, V]) map[K]V {
+ if c == nil {
+ return nil
+ }
+
+ out := make(map[K]V)
+ for _, data := range *c {
+ out[data.Key] = data.Value
+ }
+ return out
+}
+
+type dataFile struct {
+ Content ManifestEntryContent `avro:"content"`
+ Path string `avro:"file_path"`
+ Format FileFormat `avro:"file_format"`
+ PartitionData map[string]any `avro:"partition"`
+ RecordCount int64 `avro:"record_count"`
+ FileSize int64 `avro:"file_size_in_bytes"`
+ BlockSizeInBytes int64 `avro:"block_size_in_bytes"`
+ ColSizes *[]colMap[int, int64] `avro:"column_sizes"`
+ ValCounts *[]colMap[int, int64] `avro:"value_counts"`
+ NullCounts *[]colMap[int, int64] `avro:"null_value_counts"`
+ NaNCounts *[]colMap[int, int64] `avro:"nan_value_counts"`
+ DistinctCounts *[]colMap[int, int64] `avro:"distinct_counts"`
+ LowerBounds *[]colMap[int, []byte] `avro:"lower_bounds"`
+ UpperBounds *[]colMap[int, []byte] `avro:"upper_bounds"`
+ KeyMetadata *[]byte `avro:"key_metadata"`
+ SplitOffsets *[]int64 `avro:"split_offsets"`
+ EqualityIDs *[]int `avro:"equality_ids"`
+ SortOrder *int `avro:"sort_order_id"`
+
+ colSizeMap map[int]int64
+ valCntMap map[int]int64
+ nullCntMap map[int]int64
+ nanCntMap map[int]int64
+ distinctCntMap map[int]int64
+ lowerBoundMap map[int][]byte
+ upperBoundMap map[int][]byte
+
+ initMaps sync.Once
+}
+
+func (d *dataFile) initializeMapData() {
+ d.initMaps.Do(func() {
+ d.colSizeMap = avroColMapToMap(d.ColSizes)
+ d.valCntMap = avroColMapToMap(d.ValCounts)
+ d.nullCntMap = avroColMapToMap(d.NullCounts)
+ d.nanCntMap = avroColMapToMap(d.NaNCounts)
+ d.distinctCntMap = avroColMapToMap(d.DistinctCounts)
+ d.lowerBoundMap = avroColMapToMap(d.LowerBounds)
+ d.upperBoundMap = avroColMapToMap(d.UpperBounds)
+ })
+}
+
+func (d *dataFile) ContentType() ManifestEntryContent { return d.Content }
+func (d *dataFile) FilePath() string { return d.Path }
+func (d *dataFile) FileFormat() FileFormat { return d.Format }
+func (d *dataFile) Partition() map[string]any { return d.PartitionData }
+func (d *dataFile) Count() int64 { return d.RecordCount }
+func (d *dataFile) FileSizeBytes() int64 { return d.FileSize }
+
+func (d *dataFile) ColumnSizes() map[int]int64 {
+ d.initializeMapData()
+ return d.colSizeMap
+}
+
+func (d *dataFile) ValueCounts() map[int]int64 {
+ d.initializeMapData()
+ return d.valCntMap
+}
+
+func (d *dataFile) NullValueCounts() map[int]int64 {
+ d.initializeMapData()
+ return d.nullCntMap
+}
+
+func (d *dataFile) NaNValueCounts() map[int]int64 {
+ d.initializeMapData()
+ return d.nanCntMap
+}
+
+func (d *dataFile) DistinctValueCounts() map[int]int64 {
+ d.initializeMapData()
+ return d.distinctCntMap
+}
+
+func (d *dataFile) LowerBoundValues() map[int][]byte {
+ d.initializeMapData()
+ return d.lowerBoundMap
+}
+
+func (d *dataFile) UpperBoundValues() map[int][]byte {
+ d.initializeMapData()
+ return d.upperBoundMap
+}
+
+func (d *dataFile) MetadataKey() []byte {
+ if d.KeyMetadata == nil {
+ return nil
+ }
+ return *d.KeyMetadata
+}
+
+func (d *dataFile) Splits() []int64 {
+ if d.SplitOffsets == nil {
+ return nil
+ }
+ return *d.SplitOffsets
+}
+
+func (d *dataFile) EqualityFieldIDs() []int {
+ if d.EqualityIDs == nil {
+ return nil
+ }
+ return d.EqualityFieldIDs()
+}
+
+func (d *dataFile) SortOrderID() *int { return d.SortOrder }
+
+type manifestEntryV1 struct {
+ EntryStatus ManifestEntryStatus `avro:"status"`
+ Snapshot int64 `avro:"snapshot_id"`
+ SeqNum *int64
+ FileSeqNum *int64
+ Data dataFile `avro:"data_file"`
+}
+
+func (m *manifestEntryV1) inheritSeqNum(manifest ManifestFile) {}
+
+func (m *manifestEntryV1) Status() ManifestEntryStatus { return m.EntryStatus }
+func (m *manifestEntryV1) SnapshotID() int64 { return m.Snapshot }
+
+func (m *manifestEntryV1) SequenceNum() int64 {
+ if m.SeqNum == nil {
+ return 0
+ }
+ return *m.SeqNum
+}
+
+func (m *manifestEntryV1) FileSequenceNum() *int64 {
+ return m.FileSeqNum
+}
+
+func (m *manifestEntryV1) DataFile() DataFile { return &m.Data }
+
+type manifestEntryV2 struct {
+ EntryStatus ManifestEntryStatus `avro:"status"`
+ Snapshot *int64 `avro:"snapshot_id"`
+ SeqNum *int64 `avro:"sequence_number"`
+ FileSeqNum *int64 `avro:"file_sequence_number"`
+ Data dataFile `avro:"data_file"`
+}
+
+func (m *manifestEntryV2) inheritSeqNum(manifest ManifestFile) {
+ if m.Snapshot == nil {
+ snap := manifest.SnapshotID()
+ m.Snapshot = &snap
+ }
+
+ manifestSequenceNum := manifest.SequenceNum()
+ if m.SeqNum == nil && (manifestSequenceNum == 0 || m.EntryStatus == EntryStatusADDED) {
+ m.SeqNum = &manifestSequenceNum
+ }
+
+ if m.FileSeqNum == nil && (manifestSequenceNum == 0 || m.EntryStatus == EntryStatusADDED) {
+ m.FileSeqNum = &manifestSequenceNum
+ }
+}
+
+func (m *manifestEntryV2) Status() ManifestEntryStatus { return m.EntryStatus }
+func (m *manifestEntryV2) SnapshotID() int64 {
+ if m.Snapshot == nil {
+ return 0
+ }
+ return *m.Snapshot
+}
+
+func (m *manifestEntryV2) SequenceNum() int64 {
+ if m.SeqNum == nil {
+ return 0
+ }
+ return *m.SeqNum
+}
+
+func (m *manifestEntryV2) FileSequenceNum() *int64 {
+ return m.FileSeqNum
+}
+
+func (m *manifestEntryV2) DataFile() DataFile { return &m.Data }
+
+// DataFile is the interface for reading the information about a
+// given data file indicated by an entry in a manifest list.
+type DataFile interface {
+ // ContentType is the type of the content stored by the data file,
+ // either Data, Equality deletes, or Position deletes. All v1 files
+ // are Data files.
+ ContentType() ManifestEntryContent
+ // FilePath is the full URI for the file, complete with FS scheme.
+ FilePath() string
+ // FileFormat is the format of the data file, AVRO, Orc, or Parquet.
+ FileFormat() FileFormat
+ // Partition returns a mapping of field name to partition value for
+ // each of the partition spec's fields.
+ Partition() map[string]any
+ // Count returns the number of records in this file.
+ Count() int64
+ // FileSizeBytes is the total file size in bytes.
+ FileSizeBytes() int64
+ // ColumnSizes is a mapping from column id to the total size on disk
+ // of all regions that store the column. Does not include bytes
+ // necessary to read other columns, like footers. Map will be nil for
+ // row-oriented formats (avro).
+ ColumnSizes() map[int]int64
+ // ValueCounts is a mapping from column id to the number of values
+ // in the column, including null and NaN values.
+ ValueCounts() map[int]int64
+ // NullValueCounts is a mapping from column id to the number of
+ // null values in the column.
+ NullValueCounts() map[int]int64
+ // NaNValueCounts is a mapping from column id to the number of NaN
+ // values in the column.
+ NaNValueCounts() map[int]int64
+ // DistictValueCounts is a mapping from column id to the number of
+ // distinct values in the column. Distinct counts must be derived
+ // using values in the file by counting or using sketches, but not
+ // using methods like merging existing distinct counts.
+ DistinctValueCounts() map[int]int64
+ // LowerBoundValues is a mapping from column id to the lower bounded
+ // value of the column, serialized as binary. Each value in the column
+ // must be less than or requal to all non-null, non-NaN values in the
+ // column for the file.
+ LowerBoundValues() map[int][]byte
+ // UpperBoundValues is a mapping from column id to the upper bounded
+ // value of the column, serialized as binary. Each value in the column
+ // must be greater than or equal to all non-null, non-NaN values in
+ // the column for the file.
+ UpperBoundValues() map[int][]byte
+ // MetadataKey is implementation-specific key metadata for encryption.
+ MetadataKey() []byte
+ // Splits are the split offsets for the data file. For example,
+ // all row group offsets in a Parquet file. Must be sorted ascending.
+ Splits() []int64
+ // EqualityFieldIDs are used to determine row equality in equality
+ // delete files. It is required when the content type is
+ // EntryContentEqDeletes.
+ EqualityFieldIDs() []int
+ // SortOrderID returns the id representing the sort order for this
+ // file, or nil if there is no sort order.
+ SortOrderID() *int
+}
Review Comment:
does this need `DataSequenceNumber()` and `FileSequenceNumber()`? See also https://github.com/apache/iceberg/blob/ebce8538db20fd13859b6af841cf433d9423b53c/api/src/main/java/org/apache/iceberg/ContentFile.java#L130-L147
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] nastra commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1331869900
##########
manifest.go:
##########
@@ -0,0 +1,655 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package iceberg
+
+import (
+ "io"
+ "sync"
+
+ iceio "github.com/apache/iceberg-go/io"
+
+ "github.com/hamba/avro/v2/ocf"
+)
+
+// ManifestContent indicates the type of data inside of the files
+// described by a manifest. This will indicate whether the data files
+// contain active data or deleted rows.
+type ManifestContent int32
+
+const (
+ ManifestContentData ManifestContent = 0
+ ManifestContentDeletes ManifestContent = 1
+)
+
+type fieldSummary struct {
+ ContainsNull bool `avro:"contains_null"`
+ ContainsNaN *bool `avro:"contains_nan"`
+ LowerBound *[]byte `avro:"lower_bound"`
+ UpperBound *[]byte `avro:"upper_bound"`
+}
+
+type manifestFileV1 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
Review Comment:
cc @Fokko
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] zeroshade commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1328973291
##########
manifest.go:
##########
@@ -0,0 +1,655 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package iceberg
+
+import (
+ "io"
+ "sync"
+
+ iceio "github.com/apache/iceberg-go/io"
+
+ "github.com/hamba/avro/v2/ocf"
+)
+
+// ManifestContent indicates the type of data inside of the files
+// described by a manifest. This will indicate whether the data files
+// contain active data or deleted rows.
+type ManifestContent int32
+
+const (
+ ManifestContentData ManifestContent = 0
+ ManifestContentDeletes ManifestContent = 1
+)
+
+type fieldSummary struct {
+ ContainsNull bool `avro:"contains_null"`
+ ContainsNaN *bool `avro:"contains_nan"`
+ LowerBound *[]byte `avro:"lower_bound"`
+ UpperBound *[]byte `avro:"upper_bound"`
+}
+
+type manifestFileV1 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ AddedSnapshotID *int64 `avro:"added_snapshot_id"`
+ AddedFilesCount *int32 `avro:"added_data_files_count"`
+ ExistingFilesCount *int32 `avro:"existing_data_files_count"`
+ DeletedFilesCount *int32 `avro:"deleted_data_files_count"`
+ AddedRowsCount *int64 `avro:"added_rows_count"`
+ ExistingRowsCount *int64 `avro:"existing_rows_count"`
+ DeletedRowsCount *int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV1) Version() int { return 1 }
+func (m *manifestFileV1) FilePath() string { return m.Path }
+func (m *manifestFileV1) Length() int64 { return m.Len }
+func (m *manifestFileV1) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV1) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV1) SnapshotID() int64 {
+ if m.AddedSnapshotID == nil {
+ return 0
+ }
+ return *m.AddedSnapshotID
+}
+
+func (m *manifestFileV1) AddedDataFiles() int32 {
+ if m.AddedFilesCount == nil {
+ return 0
+ }
+ return *m.AddedFilesCount
+}
+
+func (m *manifestFileV1) ExistingDataFiles() int32 {
+ if m.ExistingFilesCount == nil {
+ return 0
+ }
+ return *m.ExistingFilesCount
+}
+
+func (m *manifestFileV1) DeletedDataFiles() int32 {
+ if m.DeletedFilesCount == nil {
+ return 0
+ }
+ return *m.DeletedFilesCount
+}
+
+func (m *manifestFileV1) AddedRows() int64 {
+ if m.AddedRowsCount == nil {
+ return 0
+ }
+ return *m.AddedRowsCount
+}
+
+func (m *manifestFileV1) ExistingRows() int64 {
+ if m.ExistingRowsCount == nil {
+ return 0
+ }
+ return *m.ExistingRowsCount
+}
+
+func (m *manifestFileV1) DeletedRows() int64 {
+ if m.DeletedRowsCount == nil {
+ return 0
+ }
+ return *m.DeletedRowsCount
+}
+
+func (m *manifestFileV1) HasAddedFiles() bool {
+ return m.AddedFilesCount == nil || *m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV1) HasExistingFiles() bool {
+ return m.ExistingFilesCount == nil || *m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV1) SequenceNum() int64 { return 0 }
+func (m *manifestFileV1) MinSequenceNum() int64 { return 0 }
+func (m *manifestFileV1) Metadata() []byte { return m.KeyMetadata }
+func (m *manifestFileV1) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV1) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+type manifestFileV2 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ SeqNumber int64 `avro:"sequence_number"`
+ MinSeqNumber int64 `avro:"min_sequence_number"`
+ AddedSnapshotID int64 `avro:"added_snapshot_id"`
+ AddedFilesCount int32 `avro:"added_files_count"`
+ ExistingFilesCount int32 `avro:"existing_files_count"`
+ DeletedFilesCount int32 `avro:"deleted_files_count"`
+ AddedRowsCount int64 `avro:"added_rows_count"`
+ ExistingRowsCount int64 `avro:"existing_rows_count"`
+ DeletedRowsCount int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV2) Version() int { return 2 }
+
+func (m *manifestFileV2) FilePath() string { return m.Path }
+func (m *manifestFileV2) Length() int64 { return m.Len }
+func (m *manifestFileV2) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV2) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV2) SnapshotID() int64 {
+ return m.AddedSnapshotID
+}
+
+func (m *manifestFileV2) AddedDataFiles() int32 {
+ return m.AddedFilesCount
+}
+
+func (m *manifestFileV2) ExistingDataFiles() int32 {
+ return m.ExistingFilesCount
+}
+
+func (m *manifestFileV2) DeletedDataFiles() int32 {
+ return m.DeletedFilesCount
+}
+
+func (m *manifestFileV2) AddedRows() int64 {
+ return m.AddedRowsCount
+}
+
+func (m *manifestFileV2) ExistingRows() int64 {
+ return m.ExistingRowsCount
+}
+
+func (m *manifestFileV2) DeletedRows() int64 {
+ return m.DeletedRowsCount
+}
+
+func (m *manifestFileV2) SequenceNum() int64 { return m.SeqNumber }
+func (m *manifestFileV2) MinSequenceNum() int64 { return m.MinSeqNumber }
+func (m *manifestFileV2) Metadata() []byte { return m.KeyMetadata }
+
+func (m *manifestFileV2) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV2) HasAddedFiles() bool {
+ return m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV2) HasExistingFiles() bool {
+ return m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV2) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+func fetchManifestEntries(m ManifestFile, fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ f, err := fs.Open(m.FilePath())
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+
+ dec, err := ocf.NewDecoder(f)
+ if err != nil {
+ return nil, err
+ }
+
+ metadata := dec.Metadata()
+ isVer1 := true
+ if string(metadata["format-version"]) == "2" {
+ isVer1 = false
+ }
+
+ results := make([]ManifestEntry, 0)
+ for dec.HasNext() {
+ var tmp ManifestEntry
+ if isVer1 {
+ tmp = &manifestEntryV1{}
+ } else {
+ tmp = &manifestEntryV2{}
+ }
+
+ if err := dec.Decode(tmp); err != nil {
+ return nil, err
+ }
+
+ if !discardDeleted || tmp.Status() != EntryStatusDELETED {
+ tmp.inheritSeqNum(m)
+ results = append(results, tmp)
+ }
+ }
+
+ return results, dec.Error()
+}
+
+// ManifestFile is the interface which covers both V1 and V2 manifest files.
+type ManifestFile interface {
+ // Version returns the version number of this manifest file.
+ // It should be 1 or 2.
+ Version() int
+ // FilePath is the location URI of this manifest file.
+ FilePath() string
+ // Length is the length in bytes of the manifest file.
+ Length() int64
+ // PartitionID is the ID of the partition spec used to write
+ // this manifest. It must be listed in the table metadata
+ // partition-specs.
+ PartitionID() int32
+ // ManifestContent is the type of files tracked by this manifest,
+ // either data or delete files. All v1 manifests track data files.
+ ManifestContent() ManifestContent
+ // SnapshotID is the ID of the snapshot where this manifest file
+ // was added.
+ SnapshotID() int64
+ // AddedDataFiles returns the number of entries in the manifest that
+ // have the status of EntryStatusADDED.
+ AddedDataFiles() int32
+ // ExistingDataFiles returns the number of entries in the manifest
+ // which have the status of EntryStatusEXISTING.
+ ExistingDataFiles() int32
+ // DeletedDataFiles returns the number of entries in the manifest
+ // which have the status of EntryStatusDELETED.
+ DeletedDataFiles() int32
+ // AddedRows returns the number of rows in all files of the manifest
+ // that have status EntryStatusADDED.
+ AddedRows() int64
+ // ExistingRows returns the number of rows in all files of the manifest
+ // which have status EntryStatusEXISTING.
+ ExistingRows() int64
+ // DeletedRows returns the number of rows in all files of the manifest
+ // which have status EntryStatusDELETED.
+ DeletedRows() int64
+ // SequenceNum returns the sequence number when this manifest was
+ // added to the table. Will be 0 for v1 manifest lists.
+ SequenceNum() int64
+ // MinSequenceNum is the minimum data sequence number of all live data
+ // or delete files in the manifest. Will be 0 for v1 manifest lists.
+ MinSequenceNum() int64
+ // Metadata returns implementation-specific key metadata for encryption
+ // if it exists in the manifest list.
+ Metadata() []byte
+ // PartitionList returns a list of field summaries for each partition
+ // field in the spec. Each field in the list corresponds to a field in
+ // the manifest file's partition spec.
+ PartitionList() []fieldSummary
+
+ // HasAddedFiles returns true if AddedDataFiles > 0 or if it was null.
+ HasAddedFiles() bool
+ // HasExistingFiles returns true if ExistingDataFiles > 0 or if it was null.
+ HasExistingFiles() bool
+ // FetchEntries reads the manifest list file to fetch the list of
+ // manifest entries using the provided file system IO interface.
+ // If discardDeleted is true, entries for files containing deleted rows
+ // will be skipped.
+ FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error)
+}
+
+// ReadManifestList reads in an avro manifest list file and returns a slice
+// of manifest files or an error if one is encountered.
+func ReadManifestList(in io.Reader) ([]ManifestFile, error) {
+ dec, err := ocf.NewDecoder(in)
+ if err != nil {
+ return nil, err
+ }
+
+ out := make([]ManifestFile, 0)
+
+ for dec.HasNext() {
+ var file ManifestFile
+ if string(dec.Metadata()["format-version"]) == "2" {
+ file = &manifestFileV2{}
+ } else {
+ file = &manifestFileV1{}
+ }
+
+ if err := dec.Decode(file); err != nil {
+ return nil, err
+ }
+ out = append(out, file)
+ }
+
+ return out, dec.Error()
+}
+
+// ManifestEntryStatus defines constants for the entry status of
+// existing, added or deleted.
+type ManifestEntryStatus int8
+
+const (
+ EntryStatusEXISTING ManifestEntryStatus = 0
+ EntryStatusADDED ManifestEntryStatus = 1
+ EntryStatusDELETED ManifestEntryStatus = 2
+)
+
+// ManifestEntryContent defines constants for the type of file contents
+// in the file entries. Data, Position based deletes and equality based
+// deletes.
+type ManifestEntryContent int8
+
+const (
+ EntryContentData ManifestEntryContent = 0
+ EntryContentPosDeletes ManifestEntryContent = 1
+ EntryContentEqDeletes ManifestEntryContent = 2
+)
+
+// FileFormat defines constants for the format of data files.
+type FileFormat string
+
+const (
+ AvroFile FileFormat = "AVRO"
+ OrcFile FileFormat = "ORC"
+ ParquetFile FileFormat = "PARQUET"
+)
+
+type colMap[K, V any] struct {
+ Key K `avro:"key"`
+ Value V `avro:"value"`
+}
+
+func avroColMapToMap[K comparable, V any](c *[]colMap[K, V]) map[K]V {
+ if c == nil {
+ return nil
+ }
+
+ out := make(map[K]V)
+ for _, data := range *c {
+ out[data.Key] = data.Value
+ }
+ return out
+}
+
+type dataFile struct {
+ Content ManifestEntryContent `avro:"content"`
+ Path string `avro:"file_path"`
+ Format FileFormat `avro:"file_format"`
+ PartitionData map[string]any `avro:"partition"`
+ RecordCount int64 `avro:"record_count"`
+ FileSize int64 `avro:"file_size_in_bytes"`
+ BlockSizeInBytes int64 `avro:"block_size_in_bytes"`
+ ColSizes *[]colMap[int, int64] `avro:"column_sizes"`
+ ValCounts *[]colMap[int, int64] `avro:"value_counts"`
+ NullCounts *[]colMap[int, int64] `avro:"null_value_counts"`
+ NaNCounts *[]colMap[int, int64] `avro:"nan_value_counts"`
+ DistinctCounts *[]colMap[int, int64] `avro:"distinct_counts"`
+ LowerBounds *[]colMap[int, []byte] `avro:"lower_bounds"`
+ UpperBounds *[]colMap[int, []byte] `avro:"upper_bounds"`
+ KeyMetadata *[]byte `avro:"key_metadata"`
+ SplitOffsets *[]int64 `avro:"split_offsets"`
+ EqualityIDs *[]int `avro:"equality_ids"`
Review Comment:
I know, since it is only optional in v2 i'm able to get away without needing separate data structures for V1 and V2 data files since the avro schema will ensure it would be ignored in a v1 file and just left nil. If preferred I can add a comment here to that effect. I just wanted to avoid the duplication since I don't need the separate v1/v2 data file structs.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] zeroshade commented on pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade commented on PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#issuecomment-1723930518
@nastra I've implemented the suggested fixes but I had a few questions that I posed when you get a chance. Thanks again!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] zeroshade commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1328983484
##########
manifest.go:
##########
@@ -0,0 +1,655 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package iceberg
+
+import (
+ "io"
+ "sync"
+
+ iceio "github.com/apache/iceberg-go/io"
+
+ "github.com/hamba/avro/v2/ocf"
+)
+
+// ManifestContent indicates the type of data inside of the files
+// described by a manifest. This will indicate whether the data files
+// contain active data or deleted rows.
+type ManifestContent int32
+
+const (
+ ManifestContentData ManifestContent = 0
+ ManifestContentDeletes ManifestContent = 1
+)
+
+type fieldSummary struct {
+ ContainsNull bool `avro:"contains_null"`
+ ContainsNaN *bool `avro:"contains_nan"`
+ LowerBound *[]byte `avro:"lower_bound"`
+ UpperBound *[]byte `avro:"upper_bound"`
+}
+
+type manifestFileV1 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ AddedSnapshotID *int64 `avro:"added_snapshot_id"`
+ AddedFilesCount *int32 `avro:"added_data_files_count"`
+ ExistingFilesCount *int32 `avro:"existing_data_files_count"`
+ DeletedFilesCount *int32 `avro:"deleted_data_files_count"`
+ AddedRowsCount *int64 `avro:"added_rows_count"`
+ ExistingRowsCount *int64 `avro:"existing_rows_count"`
+ DeletedRowsCount *int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV1) Version() int { return 1 }
+func (m *manifestFileV1) FilePath() string { return m.Path }
+func (m *manifestFileV1) Length() int64 { return m.Len }
+func (m *manifestFileV1) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV1) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV1) SnapshotID() int64 {
+ if m.AddedSnapshotID == nil {
+ return 0
+ }
+ return *m.AddedSnapshotID
+}
+
+func (m *manifestFileV1) AddedDataFiles() int32 {
+ if m.AddedFilesCount == nil {
+ return 0
+ }
+ return *m.AddedFilesCount
+}
+
+func (m *manifestFileV1) ExistingDataFiles() int32 {
+ if m.ExistingFilesCount == nil {
+ return 0
+ }
+ return *m.ExistingFilesCount
+}
+
+func (m *manifestFileV1) DeletedDataFiles() int32 {
+ if m.DeletedFilesCount == nil {
+ return 0
+ }
+ return *m.DeletedFilesCount
+}
+
+func (m *manifestFileV1) AddedRows() int64 {
+ if m.AddedRowsCount == nil {
+ return 0
+ }
+ return *m.AddedRowsCount
+}
+
+func (m *manifestFileV1) ExistingRows() int64 {
+ if m.ExistingRowsCount == nil {
+ return 0
+ }
+ return *m.ExistingRowsCount
+}
+
+func (m *manifestFileV1) DeletedRows() int64 {
+ if m.DeletedRowsCount == nil {
+ return 0
+ }
+ return *m.DeletedRowsCount
+}
+
+func (m *manifestFileV1) HasAddedFiles() bool {
+ return m.AddedFilesCount == nil || *m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV1) HasExistingFiles() bool {
+ return m.ExistingFilesCount == nil || *m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV1) SequenceNum() int64 { return 0 }
+func (m *manifestFileV1) MinSequenceNum() int64 { return 0 }
+func (m *manifestFileV1) Metadata() []byte { return m.KeyMetadata }
+func (m *manifestFileV1) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV1) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+type manifestFileV2 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ SeqNumber int64 `avro:"sequence_number"`
+ MinSeqNumber int64 `avro:"min_sequence_number"`
+ AddedSnapshotID int64 `avro:"added_snapshot_id"`
+ AddedFilesCount int32 `avro:"added_files_count"`
+ ExistingFilesCount int32 `avro:"existing_files_count"`
+ DeletedFilesCount int32 `avro:"deleted_files_count"`
+ AddedRowsCount int64 `avro:"added_rows_count"`
+ ExistingRowsCount int64 `avro:"existing_rows_count"`
+ DeletedRowsCount int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV2) Version() int { return 2 }
+
+func (m *manifestFileV2) FilePath() string { return m.Path }
+func (m *manifestFileV2) Length() int64 { return m.Len }
+func (m *manifestFileV2) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV2) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV2) SnapshotID() int64 {
+ return m.AddedSnapshotID
+}
+
+func (m *manifestFileV2) AddedDataFiles() int32 {
+ return m.AddedFilesCount
+}
+
+func (m *manifestFileV2) ExistingDataFiles() int32 {
+ return m.ExistingFilesCount
+}
+
+func (m *manifestFileV2) DeletedDataFiles() int32 {
+ return m.DeletedFilesCount
+}
+
+func (m *manifestFileV2) AddedRows() int64 {
+ return m.AddedRowsCount
+}
+
+func (m *manifestFileV2) ExistingRows() int64 {
+ return m.ExistingRowsCount
+}
+
+func (m *manifestFileV2) DeletedRows() int64 {
+ return m.DeletedRowsCount
+}
+
+func (m *manifestFileV2) SequenceNum() int64 { return m.SeqNumber }
+func (m *manifestFileV2) MinSequenceNum() int64 { return m.MinSeqNumber }
+func (m *manifestFileV2) Metadata() []byte { return m.KeyMetadata }
+
+func (m *manifestFileV2) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV2) HasAddedFiles() bool {
+ return m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV2) HasExistingFiles() bool {
+ return m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV2) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+func fetchManifestEntries(m ManifestFile, fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ f, err := fs.Open(m.FilePath())
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+
+ dec, err := ocf.NewDecoder(f)
+ if err != nil {
+ return nil, err
+ }
+
+ metadata := dec.Metadata()
+ isVer1 := true
+ if string(metadata["format-version"]) == "2" {
+ isVer1 = false
+ }
+
+ results := make([]ManifestEntry, 0)
+ for dec.HasNext() {
+ var tmp ManifestEntry
+ if isVer1 {
+ tmp = &manifestEntryV1{}
+ } else {
+ tmp = &manifestEntryV2{}
+ }
+
+ if err := dec.Decode(tmp); err != nil {
+ return nil, err
+ }
+
+ if !discardDeleted || tmp.Status() != EntryStatusDELETED {
+ tmp.inheritSeqNum(m)
+ results = append(results, tmp)
+ }
+ }
+
+ return results, dec.Error()
+}
+
+// ManifestFile is the interface which covers both V1 and V2 manifest files.
+type ManifestFile interface {
+ // Version returns the version number of this manifest file.
+ // It should be 1 or 2.
+ Version() int
+ // FilePath is the location URI of this manifest file.
+ FilePath() string
+ // Length is the length in bytes of the manifest file.
+ Length() int64
+ // PartitionID is the ID of the partition spec used to write
+ // this manifest. It must be listed in the table metadata
+ // partition-specs.
+ PartitionID() int32
+ // ManifestContent is the type of files tracked by this manifest,
+ // either data or delete files. All v1 manifests track data files.
+ ManifestContent() ManifestContent
+ // SnapshotID is the ID of the snapshot where this manifest file
+ // was added.
+ SnapshotID() int64
+ // AddedDataFiles returns the number of entries in the manifest that
+ // have the status of EntryStatusADDED.
+ AddedDataFiles() int32
+ // ExistingDataFiles returns the number of entries in the manifest
+ // which have the status of EntryStatusEXISTING.
+ ExistingDataFiles() int32
+ // DeletedDataFiles returns the number of entries in the manifest
+ // which have the status of EntryStatusDELETED.
+ DeletedDataFiles() int32
+ // AddedRows returns the number of rows in all files of the manifest
+ // that have status EntryStatusADDED.
+ AddedRows() int64
+ // ExistingRows returns the number of rows in all files of the manifest
+ // which have status EntryStatusEXISTING.
+ ExistingRows() int64
+ // DeletedRows returns the number of rows in all files of the manifest
+ // which have status EntryStatusDELETED.
+ DeletedRows() int64
+ // SequenceNum returns the sequence number when this manifest was
+ // added to the table. Will be 0 for v1 manifest lists.
+ SequenceNum() int64
+ // MinSequenceNum is the minimum data sequence number of all live data
+ // or delete files in the manifest. Will be 0 for v1 manifest lists.
+ MinSequenceNum() int64
+ // Metadata returns implementation-specific key metadata for encryption
+ // if it exists in the manifest list.
+ Metadata() []byte
+ // PartitionList returns a list of field summaries for each partition
+ // field in the spec. Each field in the list corresponds to a field in
+ // the manifest file's partition spec.
+ PartitionList() []fieldSummary
+
+ // HasAddedFiles returns true if AddedDataFiles > 0 or if it was null.
+ HasAddedFiles() bool
+ // HasExistingFiles returns true if ExistingDataFiles > 0 or if it was null.
+ HasExistingFiles() bool
+ // FetchEntries reads the manifest list file to fetch the list of
+ // manifest entries using the provided file system IO interface.
+ // If discardDeleted is true, entries for files containing deleted rows
+ // will be skipped.
+ FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error)
+}
+
+// ReadManifestList reads in an avro manifest list file and returns a slice
+// of manifest files or an error if one is encountered.
+func ReadManifestList(in io.Reader) ([]ManifestFile, error) {
+ dec, err := ocf.NewDecoder(in)
+ if err != nil {
+ return nil, err
+ }
+
+ out := make([]ManifestFile, 0)
+
+ for dec.HasNext() {
+ var file ManifestFile
+ if string(dec.Metadata()["format-version"]) == "2" {
+ file = &manifestFileV2{}
+ } else {
+ file = &manifestFileV1{}
+ }
+
+ if err := dec.Decode(file); err != nil {
+ return nil, err
+ }
+ out = append(out, file)
+ }
+
+ return out, dec.Error()
+}
+
+// ManifestEntryStatus defines constants for the entry status of
+// existing, added or deleted.
+type ManifestEntryStatus int8
+
+const (
+ EntryStatusEXISTING ManifestEntryStatus = 0
+ EntryStatusADDED ManifestEntryStatus = 1
+ EntryStatusDELETED ManifestEntryStatus = 2
+)
+
+// ManifestEntryContent defines constants for the type of file contents
+// in the file entries. Data, Position based deletes and equality based
+// deletes.
+type ManifestEntryContent int8
+
+const (
+ EntryContentData ManifestEntryContent = 0
+ EntryContentPosDeletes ManifestEntryContent = 1
+ EntryContentEqDeletes ManifestEntryContent = 2
+)
+
+// FileFormat defines constants for the format of data files.
+type FileFormat string
+
+const (
+ AvroFile FileFormat = "AVRO"
+ OrcFile FileFormat = "ORC"
+ ParquetFile FileFormat = "PARQUET"
+)
+
+type colMap[K, V any] struct {
+ Key K `avro:"key"`
+ Value V `avro:"value"`
+}
+
+func avroColMapToMap[K comparable, V any](c *[]colMap[K, V]) map[K]V {
+ if c == nil {
+ return nil
+ }
+
+ out := make(map[K]V)
+ for _, data := range *c {
+ out[data.Key] = data.Value
+ }
+ return out
+}
+
+type dataFile struct {
+ Content ManifestEntryContent `avro:"content"`
+ Path string `avro:"file_path"`
+ Format FileFormat `avro:"file_format"`
+ PartitionData map[string]any `avro:"partition"`
+ RecordCount int64 `avro:"record_count"`
+ FileSize int64 `avro:"file_size_in_bytes"`
+ BlockSizeInBytes int64 `avro:"block_size_in_bytes"`
Review Comment:
Since the fields don't exist in the defined schema for V2 they shouldn't get written when writing the file since we write according to the schema
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] zeroshade commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1328975024
##########
manifest.go:
##########
@@ -0,0 +1,655 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package iceberg
+
+import (
+ "io"
+ "sync"
+
+ iceio "github.com/apache/iceberg-go/io"
+
+ "github.com/hamba/avro/v2/ocf"
+)
+
+// ManifestContent indicates the type of data inside of the files
+// described by a manifest. This will indicate whether the data files
+// contain active data or deleted rows.
+type ManifestContent int32
+
+const (
+ ManifestContentData ManifestContent = 0
+ ManifestContentDeletes ManifestContent = 1
+)
+
+type fieldSummary struct {
+ ContainsNull bool `avro:"contains_null"`
+ ContainsNaN *bool `avro:"contains_nan"`
+ LowerBound *[]byte `avro:"lower_bound"`
+ UpperBound *[]byte `avro:"upper_bound"`
+}
+
+type manifestFileV1 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ AddedSnapshotID *int64 `avro:"added_snapshot_id"`
+ AddedFilesCount *int32 `avro:"added_data_files_count"`
+ ExistingFilesCount *int32 `avro:"existing_data_files_count"`
+ DeletedFilesCount *int32 `avro:"deleted_data_files_count"`
+ AddedRowsCount *int64 `avro:"added_rows_count"`
+ ExistingRowsCount *int64 `avro:"existing_rows_count"`
+ DeletedRowsCount *int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV1) Version() int { return 1 }
+func (m *manifestFileV1) FilePath() string { return m.Path }
+func (m *manifestFileV1) Length() int64 { return m.Len }
+func (m *manifestFileV1) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV1) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV1) SnapshotID() int64 {
+ if m.AddedSnapshotID == nil {
+ return 0
+ }
+ return *m.AddedSnapshotID
+}
+
+func (m *manifestFileV1) AddedDataFiles() int32 {
+ if m.AddedFilesCount == nil {
+ return 0
+ }
+ return *m.AddedFilesCount
+}
+
+func (m *manifestFileV1) ExistingDataFiles() int32 {
+ if m.ExistingFilesCount == nil {
+ return 0
+ }
+ return *m.ExistingFilesCount
+}
+
+func (m *manifestFileV1) DeletedDataFiles() int32 {
+ if m.DeletedFilesCount == nil {
+ return 0
+ }
+ return *m.DeletedFilesCount
+}
+
+func (m *manifestFileV1) AddedRows() int64 {
+ if m.AddedRowsCount == nil {
+ return 0
+ }
+ return *m.AddedRowsCount
+}
+
+func (m *manifestFileV1) ExistingRows() int64 {
+ if m.ExistingRowsCount == nil {
+ return 0
+ }
+ return *m.ExistingRowsCount
+}
+
+func (m *manifestFileV1) DeletedRows() int64 {
+ if m.DeletedRowsCount == nil {
+ return 0
+ }
+ return *m.DeletedRowsCount
+}
+
+func (m *manifestFileV1) HasAddedFiles() bool {
+ return m.AddedFilesCount == nil || *m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV1) HasExistingFiles() bool {
+ return m.ExistingFilesCount == nil || *m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV1) SequenceNum() int64 { return 0 }
+func (m *manifestFileV1) MinSequenceNum() int64 { return 0 }
+func (m *manifestFileV1) Metadata() []byte { return m.KeyMetadata }
+func (m *manifestFileV1) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV1) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+type manifestFileV2 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ SeqNumber int64 `avro:"sequence_number"`
+ MinSeqNumber int64 `avro:"min_sequence_number"`
+ AddedSnapshotID int64 `avro:"added_snapshot_id"`
+ AddedFilesCount int32 `avro:"added_files_count"`
+ ExistingFilesCount int32 `avro:"existing_files_count"`
+ DeletedFilesCount int32 `avro:"deleted_files_count"`
+ AddedRowsCount int64 `avro:"added_rows_count"`
+ ExistingRowsCount int64 `avro:"existing_rows_count"`
+ DeletedRowsCount int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV2) Version() int { return 2 }
+
+func (m *manifestFileV2) FilePath() string { return m.Path }
+func (m *manifestFileV2) Length() int64 { return m.Len }
+func (m *manifestFileV2) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV2) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV2) SnapshotID() int64 {
+ return m.AddedSnapshotID
+}
+
+func (m *manifestFileV2) AddedDataFiles() int32 {
+ return m.AddedFilesCount
+}
+
+func (m *manifestFileV2) ExistingDataFiles() int32 {
+ return m.ExistingFilesCount
+}
+
+func (m *manifestFileV2) DeletedDataFiles() int32 {
+ return m.DeletedFilesCount
+}
+
+func (m *manifestFileV2) AddedRows() int64 {
+ return m.AddedRowsCount
+}
+
+func (m *manifestFileV2) ExistingRows() int64 {
+ return m.ExistingRowsCount
+}
+
+func (m *manifestFileV2) DeletedRows() int64 {
+ return m.DeletedRowsCount
+}
+
+func (m *manifestFileV2) SequenceNum() int64 { return m.SeqNumber }
+func (m *manifestFileV2) MinSequenceNum() int64 { return m.MinSeqNumber }
+func (m *manifestFileV2) Metadata() []byte { return m.KeyMetadata }
+
+func (m *manifestFileV2) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV2) HasAddedFiles() bool {
+ return m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV2) HasExistingFiles() bool {
+ return m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV2) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+func fetchManifestEntries(m ManifestFile, fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ f, err := fs.Open(m.FilePath())
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+
+ dec, err := ocf.NewDecoder(f)
+ if err != nil {
+ return nil, err
+ }
+
+ metadata := dec.Metadata()
+ isVer1 := true
+ if string(metadata["format-version"]) == "2" {
+ isVer1 = false
+ }
+
+ results := make([]ManifestEntry, 0)
+ for dec.HasNext() {
+ var tmp ManifestEntry
+ if isVer1 {
+ tmp = &manifestEntryV1{}
+ } else {
+ tmp = &manifestEntryV2{}
+ }
+
+ if err := dec.Decode(tmp); err != nil {
+ return nil, err
+ }
+
+ if !discardDeleted || tmp.Status() != EntryStatusDELETED {
+ tmp.inheritSeqNum(m)
+ results = append(results, tmp)
+ }
+ }
+
+ return results, dec.Error()
+}
+
+// ManifestFile is the interface which covers both V1 and V2 manifest files.
+type ManifestFile interface {
+ // Version returns the version number of this manifest file.
+ // It should be 1 or 2.
+ Version() int
+ // FilePath is the location URI of this manifest file.
+ FilePath() string
+ // Length is the length in bytes of the manifest file.
+ Length() int64
+ // PartitionID is the ID of the partition spec used to write
+ // this manifest. It must be listed in the table metadata
+ // partition-specs.
+ PartitionID() int32
+ // ManifestContent is the type of files tracked by this manifest,
+ // either data or delete files. All v1 manifests track data files.
+ ManifestContent() ManifestContent
+ // SnapshotID is the ID of the snapshot where this manifest file
+ // was added.
+ SnapshotID() int64
+ // AddedDataFiles returns the number of entries in the manifest that
+ // have the status of EntryStatusADDED.
+ AddedDataFiles() int32
+ // ExistingDataFiles returns the number of entries in the manifest
+ // which have the status of EntryStatusEXISTING.
+ ExistingDataFiles() int32
+ // DeletedDataFiles returns the number of entries in the manifest
+ // which have the status of EntryStatusDELETED.
+ DeletedDataFiles() int32
+ // AddedRows returns the number of rows in all files of the manifest
+ // that have status EntryStatusADDED.
+ AddedRows() int64
+ // ExistingRows returns the number of rows in all files of the manifest
+ // which have status EntryStatusEXISTING.
+ ExistingRows() int64
+ // DeletedRows returns the number of rows in all files of the manifest
+ // which have status EntryStatusDELETED.
+ DeletedRows() int64
+ // SequenceNum returns the sequence number when this manifest was
+ // added to the table. Will be 0 for v1 manifest lists.
+ SequenceNum() int64
+ // MinSequenceNum is the minimum data sequence number of all live data
+ // or delete files in the manifest. Will be 0 for v1 manifest lists.
+ MinSequenceNum() int64
+ // Metadata returns implementation-specific key metadata for encryption
+ // if it exists in the manifest list.
+ Metadata() []byte
+ // PartitionList returns a list of field summaries for each partition
+ // field in the spec. Each field in the list corresponds to a field in
+ // the manifest file's partition spec.
+ PartitionList() []fieldSummary
+
+ // HasAddedFiles returns true if AddedDataFiles > 0 or if it was null.
+ HasAddedFiles() bool
+ // HasExistingFiles returns true if ExistingDataFiles > 0 or if it was null.
+ HasExistingFiles() bool
+ // FetchEntries reads the manifest list file to fetch the list of
+ // manifest entries using the provided file system IO interface.
+ // If discardDeleted is true, entries for files containing deleted rows
+ // will be skipped.
+ FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error)
+}
+
+// ReadManifestList reads in an avro manifest list file and returns a slice
+// of manifest files or an error if one is encountered.
+func ReadManifestList(in io.Reader) ([]ManifestFile, error) {
+ dec, err := ocf.NewDecoder(in)
+ if err != nil {
+ return nil, err
+ }
+
+ out := make([]ManifestFile, 0)
+
+ for dec.HasNext() {
+ var file ManifestFile
+ if string(dec.Metadata()["format-version"]) == "2" {
+ file = &manifestFileV2{}
+ } else {
+ file = &manifestFileV1{}
+ }
+
+ if err := dec.Decode(file); err != nil {
+ return nil, err
+ }
+ out = append(out, file)
+ }
+
+ return out, dec.Error()
+}
+
+// ManifestEntryStatus defines constants for the entry status of
+// existing, added or deleted.
+type ManifestEntryStatus int8
+
+const (
+ EntryStatusEXISTING ManifestEntryStatus = 0
+ EntryStatusADDED ManifestEntryStatus = 1
+ EntryStatusDELETED ManifestEntryStatus = 2
+)
+
+// ManifestEntryContent defines constants for the type of file contents
+// in the file entries. Data, Position based deletes and equality based
+// deletes.
+type ManifestEntryContent int8
+
+const (
+ EntryContentData ManifestEntryContent = 0
+ EntryContentPosDeletes ManifestEntryContent = 1
+ EntryContentEqDeletes ManifestEntryContent = 2
+)
+
+// FileFormat defines constants for the format of data files.
+type FileFormat string
+
+const (
+ AvroFile FileFormat = "AVRO"
+ OrcFile FileFormat = "ORC"
+ ParquetFile FileFormat = "PARQUET"
+)
+
+type colMap[K, V any] struct {
+ Key K `avro:"key"`
+ Value V `avro:"value"`
+}
+
+func avroColMapToMap[K comparable, V any](c *[]colMap[K, V]) map[K]V {
+ if c == nil {
+ return nil
+ }
+
+ out := make(map[K]V)
+ for _, data := range *c {
+ out[data.Key] = data.Value
+ }
+ return out
+}
+
+type dataFile struct {
+ Content ManifestEntryContent `avro:"content"`
Review Comment:
Is it a bad thing for it to get written for a v1 file? (readers would just ignore the field, right?)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] nastra commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1326945769
##########
io/s3.go:
##########
@@ -0,0 +1,108 @@
+// Licensed to the Apache Software Foundation (ASF) under one
Review Comment:
overall this file LGTM. what would be great is to have some tests that use S3 against a minio container, which can be done in a follow-up PR
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] nastra commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1326998314
##########
manifest.go:
##########
@@ -0,0 +1,655 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package iceberg
+
+import (
+ "io"
+ "sync"
+
+ iceio "github.com/apache/iceberg-go/io"
+
+ "github.com/hamba/avro/v2/ocf"
+)
+
+// ManifestContent indicates the type of data inside of the files
+// described by a manifest. This will indicate whether the data files
+// contain active data or deleted rows.
+type ManifestContent int32
+
+const (
+ ManifestContentData ManifestContent = 0
+ ManifestContentDeletes ManifestContent = 1
+)
+
+type fieldSummary struct {
+ ContainsNull bool `avro:"contains_null"`
+ ContainsNaN *bool `avro:"contains_nan"`
+ LowerBound *[]byte `avro:"lower_bound"`
+ UpperBound *[]byte `avro:"upper_bound"`
+}
+
+type manifestFileV1 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ AddedSnapshotID *int64 `avro:"added_snapshot_id"`
+ AddedFilesCount *int32 `avro:"added_data_files_count"`
+ ExistingFilesCount *int32 `avro:"existing_data_files_count"`
+ DeletedFilesCount *int32 `avro:"deleted_data_files_count"`
+ AddedRowsCount *int64 `avro:"added_rows_count"`
+ ExistingRowsCount *int64 `avro:"existing_rows_count"`
+ DeletedRowsCount *int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV1) Version() int { return 1 }
+func (m *manifestFileV1) FilePath() string { return m.Path }
+func (m *manifestFileV1) Length() int64 { return m.Len }
+func (m *manifestFileV1) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV1) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV1) SnapshotID() int64 {
+ if m.AddedSnapshotID == nil {
+ return 0
+ }
+ return *m.AddedSnapshotID
+}
+
+func (m *manifestFileV1) AddedDataFiles() int32 {
+ if m.AddedFilesCount == nil {
+ return 0
+ }
+ return *m.AddedFilesCount
+}
+
+func (m *manifestFileV1) ExistingDataFiles() int32 {
+ if m.ExistingFilesCount == nil {
+ return 0
+ }
+ return *m.ExistingFilesCount
+}
+
+func (m *manifestFileV1) DeletedDataFiles() int32 {
+ if m.DeletedFilesCount == nil {
+ return 0
+ }
+ return *m.DeletedFilesCount
+}
+
+func (m *manifestFileV1) AddedRows() int64 {
+ if m.AddedRowsCount == nil {
+ return 0
+ }
+ return *m.AddedRowsCount
+}
+
+func (m *manifestFileV1) ExistingRows() int64 {
+ if m.ExistingRowsCount == nil {
+ return 0
+ }
+ return *m.ExistingRowsCount
+}
+
+func (m *manifestFileV1) DeletedRows() int64 {
+ if m.DeletedRowsCount == nil {
+ return 0
+ }
+ return *m.DeletedRowsCount
+}
+
+func (m *manifestFileV1) HasAddedFiles() bool {
+ return m.AddedFilesCount == nil || *m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV1) HasExistingFiles() bool {
+ return m.ExistingFilesCount == nil || *m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV1) SequenceNum() int64 { return 0 }
+func (m *manifestFileV1) MinSequenceNum() int64 { return 0 }
+func (m *manifestFileV1) Metadata() []byte { return m.KeyMetadata }
+func (m *manifestFileV1) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV1) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+type manifestFileV2 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ SeqNumber int64 `avro:"sequence_number"`
+ MinSeqNumber int64 `avro:"min_sequence_number"`
+ AddedSnapshotID int64 `avro:"added_snapshot_id"`
+ AddedFilesCount int32 `avro:"added_files_count"`
+ ExistingFilesCount int32 `avro:"existing_files_count"`
+ DeletedFilesCount int32 `avro:"deleted_files_count"`
+ AddedRowsCount int64 `avro:"added_rows_count"`
+ ExistingRowsCount int64 `avro:"existing_rows_count"`
+ DeletedRowsCount int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV2) Version() int { return 2 }
+
+func (m *manifestFileV2) FilePath() string { return m.Path }
+func (m *manifestFileV2) Length() int64 { return m.Len }
+func (m *manifestFileV2) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV2) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV2) SnapshotID() int64 {
+ return m.AddedSnapshotID
+}
+
+func (m *manifestFileV2) AddedDataFiles() int32 {
+ return m.AddedFilesCount
+}
+
+func (m *manifestFileV2) ExistingDataFiles() int32 {
+ return m.ExistingFilesCount
+}
+
+func (m *manifestFileV2) DeletedDataFiles() int32 {
+ return m.DeletedFilesCount
+}
+
+func (m *manifestFileV2) AddedRows() int64 {
+ return m.AddedRowsCount
+}
+
+func (m *manifestFileV2) ExistingRows() int64 {
+ return m.ExistingRowsCount
+}
+
+func (m *manifestFileV2) DeletedRows() int64 {
+ return m.DeletedRowsCount
+}
+
+func (m *manifestFileV2) SequenceNum() int64 { return m.SeqNumber }
+func (m *manifestFileV2) MinSequenceNum() int64 { return m.MinSeqNumber }
+func (m *manifestFileV2) Metadata() []byte { return m.KeyMetadata }
+
+func (m *manifestFileV2) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV2) HasAddedFiles() bool {
+ return m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV2) HasExistingFiles() bool {
+ return m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV2) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+func fetchManifestEntries(m ManifestFile, fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ f, err := fs.Open(m.FilePath())
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+
+ dec, err := ocf.NewDecoder(f)
+ if err != nil {
+ return nil, err
+ }
+
+ metadata := dec.Metadata()
+ isVer1 := true
+ if string(metadata["format-version"]) == "2" {
+ isVer1 = false
+ }
+
+ results := make([]ManifestEntry, 0)
+ for dec.HasNext() {
+ var tmp ManifestEntry
+ if isVer1 {
+ tmp = &manifestEntryV1{}
+ } else {
+ tmp = &manifestEntryV2{}
+ }
+
+ if err := dec.Decode(tmp); err != nil {
+ return nil, err
+ }
+
+ if !discardDeleted || tmp.Status() != EntryStatusDELETED {
+ tmp.inheritSeqNum(m)
+ results = append(results, tmp)
+ }
+ }
+
+ return results, dec.Error()
+}
+
+// ManifestFile is the interface which covers both V1 and V2 manifest files.
+type ManifestFile interface {
+ // Version returns the version number of this manifest file.
+ // It should be 1 or 2.
+ Version() int
+ // FilePath is the location URI of this manifest file.
+ FilePath() string
+ // Length is the length in bytes of the manifest file.
+ Length() int64
+ // PartitionID is the ID of the partition spec used to write
+ // this manifest. It must be listed in the table metadata
+ // partition-specs.
+ PartitionID() int32
+ // ManifestContent is the type of files tracked by this manifest,
+ // either data or delete files. All v1 manifests track data files.
+ ManifestContent() ManifestContent
+ // SnapshotID is the ID of the snapshot where this manifest file
+ // was added.
+ SnapshotID() int64
+ // AddedDataFiles returns the number of entries in the manifest that
+ // have the status of EntryStatusADDED.
+ AddedDataFiles() int32
+ // ExistingDataFiles returns the number of entries in the manifest
+ // which have the status of EntryStatusEXISTING.
+ ExistingDataFiles() int32
+ // DeletedDataFiles returns the number of entries in the manifest
+ // which have the status of EntryStatusDELETED.
+ DeletedDataFiles() int32
+ // AddedRows returns the number of rows in all files of the manifest
+ // that have status EntryStatusADDED.
+ AddedRows() int64
+ // ExistingRows returns the number of rows in all files of the manifest
+ // which have status EntryStatusEXISTING.
+ ExistingRows() int64
+ // DeletedRows returns the number of rows in all files of the manifest
+ // which have status EntryStatusDELETED.
+ DeletedRows() int64
+ // SequenceNum returns the sequence number when this manifest was
+ // added to the table. Will be 0 for v1 manifest lists.
+ SequenceNum() int64
+ // MinSequenceNum is the minimum data sequence number of all live data
+ // or delete files in the manifest. Will be 0 for v1 manifest lists.
+ MinSequenceNum() int64
+ // Metadata returns implementation-specific key metadata for encryption
+ // if it exists in the manifest list.
+ Metadata() []byte
+ // PartitionList returns a list of field summaries for each partition
+ // field in the spec. Each field in the list corresponds to a field in
+ // the manifest file's partition spec.
+ PartitionList() []fieldSummary
+
+ // HasAddedFiles returns true if AddedDataFiles > 0 or if it was null.
+ HasAddedFiles() bool
+ // HasExistingFiles returns true if ExistingDataFiles > 0 or if it was null.
+ HasExistingFiles() bool
+ // FetchEntries reads the manifest list file to fetch the list of
+ // manifest entries using the provided file system IO interface.
+ // If discardDeleted is true, entries for files containing deleted rows
+ // will be skipped.
+ FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error)
+}
+
+// ReadManifestList reads in an avro manifest list file and returns a slice
+// of manifest files or an error if one is encountered.
+func ReadManifestList(in io.Reader) ([]ManifestFile, error) {
+ dec, err := ocf.NewDecoder(in)
+ if err != nil {
+ return nil, err
+ }
+
+ out := make([]ManifestFile, 0)
+
+ for dec.HasNext() {
+ var file ManifestFile
+ if string(dec.Metadata()["format-version"]) == "2" {
+ file = &manifestFileV2{}
+ } else {
+ file = &manifestFileV1{}
+ }
+
+ if err := dec.Decode(file); err != nil {
+ return nil, err
+ }
+ out = append(out, file)
+ }
+
+ return out, dec.Error()
+}
+
+// ManifestEntryStatus defines constants for the entry status of
+// existing, added or deleted.
+type ManifestEntryStatus int8
+
+const (
+ EntryStatusEXISTING ManifestEntryStatus = 0
+ EntryStatusADDED ManifestEntryStatus = 1
+ EntryStatusDELETED ManifestEntryStatus = 2
+)
+
+// ManifestEntryContent defines constants for the type of file contents
+// in the file entries. Data, Position based deletes and equality based
+// deletes.
+type ManifestEntryContent int8
+
+const (
+ EntryContentData ManifestEntryContent = 0
+ EntryContentPosDeletes ManifestEntryContent = 1
+ EntryContentEqDeletes ManifestEntryContent = 2
+)
+
+// FileFormat defines constants for the format of data files.
+type FileFormat string
+
+const (
+ AvroFile FileFormat = "AVRO"
+ OrcFile FileFormat = "ORC"
+ ParquetFile FileFormat = "PARQUET"
+)
+
+type colMap[K, V any] struct {
+ Key K `avro:"key"`
+ Value V `avro:"value"`
+}
+
+func avroColMapToMap[K comparable, V any](c *[]colMap[K, V]) map[K]V {
+ if c == nil {
+ return nil
+ }
+
+ out := make(map[K]V)
+ for _, data := range *c {
+ out[data.Key] = data.Value
+ }
+ return out
+}
+
+type dataFile struct {
+ Content ManifestEntryContent `avro:"content"`
+ Path string `avro:"file_path"`
+ Format FileFormat `avro:"file_format"`
+ PartitionData map[string]any `avro:"partition"`
+ RecordCount int64 `avro:"record_count"`
+ FileSize int64 `avro:"file_size_in_bytes"`
+ BlockSizeInBytes int64 `avro:"block_size_in_bytes"`
+ ColSizes *[]colMap[int, int64] `avro:"column_sizes"`
+ ValCounts *[]colMap[int, int64] `avro:"value_counts"`
+ NullCounts *[]colMap[int, int64] `avro:"null_value_counts"`
+ NaNCounts *[]colMap[int, int64] `avro:"nan_value_counts"`
+ DistinctCounts *[]colMap[int, int64] `avro:"distinct_counts"`
+ LowerBounds *[]colMap[int, []byte] `avro:"lower_bounds"`
+ UpperBounds *[]colMap[int, []byte] `avro:"upper_bounds"`
+ KeyMetadata *[]byte `avro:"key_metadata"`
+ SplitOffsets *[]int64 `avro:"split_offsets"`
+ EqualityIDs *[]int `avro:"equality_ids"`
+ SortOrder *int `avro:"sort_order_id"`
+
+ colSizeMap map[int]int64
+ valCntMap map[int]int64
+ nullCntMap map[int]int64
+ nanCntMap map[int]int64
+ distinctCntMap map[int]int64
+ lowerBoundMap map[int][]byte
+ upperBoundMap map[int][]byte
+
+ initMaps sync.Once
+}
+
+func (d *dataFile) initializeMapData() {
+ d.initMaps.Do(func() {
+ d.colSizeMap = avroColMapToMap(d.ColSizes)
+ d.valCntMap = avroColMapToMap(d.ValCounts)
+ d.nullCntMap = avroColMapToMap(d.NullCounts)
+ d.nanCntMap = avroColMapToMap(d.NaNCounts)
+ d.distinctCntMap = avroColMapToMap(d.DistinctCounts)
+ d.lowerBoundMap = avroColMapToMap(d.LowerBounds)
+ d.upperBoundMap = avroColMapToMap(d.UpperBounds)
+ })
+}
+
+func (d *dataFile) ContentType() ManifestEntryContent { return d.Content }
+func (d *dataFile) FilePath() string { return d.Path }
+func (d *dataFile) FileFormat() FileFormat { return d.Format }
+func (d *dataFile) Partition() map[string]any { return d.PartitionData }
+func (d *dataFile) Count() int64 { return d.RecordCount }
+func (d *dataFile) FileSizeBytes() int64 { return d.FileSize }
+
+func (d *dataFile) ColumnSizes() map[int]int64 {
+ d.initializeMapData()
+ return d.colSizeMap
+}
+
+func (d *dataFile) ValueCounts() map[int]int64 {
+ d.initializeMapData()
+ return d.valCntMap
+}
+
+func (d *dataFile) NullValueCounts() map[int]int64 {
+ d.initializeMapData()
+ return d.nullCntMap
+}
+
+func (d *dataFile) NaNValueCounts() map[int]int64 {
+ d.initializeMapData()
+ return d.nanCntMap
+}
+
+func (d *dataFile) DistinctValueCounts() map[int]int64 {
+ d.initializeMapData()
+ return d.distinctCntMap
+}
+
+func (d *dataFile) LowerBoundValues() map[int][]byte {
+ d.initializeMapData()
+ return d.lowerBoundMap
+}
+
+func (d *dataFile) UpperBoundValues() map[int][]byte {
+ d.initializeMapData()
+ return d.upperBoundMap
+}
+
+func (d *dataFile) MetadataKey() []byte {
Review Comment:
```suggestion
func (d *dataFile) KeyMetadata() []byte {
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] nastra commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1326936976
##########
io/s3.go:
##########
@@ -0,0 +1,108 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package io
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "net/url"
+ "os"
+ "strings"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
+ "github.com/aws/aws-sdk-go-v2/config"
+ "github.com/aws/aws-sdk-go-v2/credentials"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+ "github.com/wolfeidau/s3iofs"
+)
+
+// Constants for S3 configuration options
+const (
+ S3Region = "s3.region"
+ S3SessionToken = "s3.session-token"
+ S3SecretAccessKey = "s3.secret-access-key"
+ S3AccessKeyID = "s3.access-key-id"
+ S3EndpointURL = "s3.endpoint"
+ S3ProxyURI = "s3.proxy-uri"
+)
+
+func createS3FileIO(parsed *url.URL, props map[string]string) (IO, error) {
+ opts := []func(*config.LoadOptions) error{}
+ endpoint, ok := props[S3EndpointURL]
+ if !ok {
+ endpoint = os.Getenv("AWS_S3_ENDPOINT")
+ }
+
+ if endpoint != "" {
+ opts = append(opts, config.WithEndpointResolverWithOptions(aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
+ if service != s3.ServiceID {
+ // fallback to default resolution for the service
+ return aws.Endpoint{}, &aws.EndpointNotFoundError{}
+ }
+
+ return aws.Endpoint{
+ URL: endpoint,
+ SigningRegion: region,
+ HostnameImmutable: true,
+ }, nil
+ })))
+ }
+
+ if defaultRegion, ok := props[S3Region]; ok {
+ opts = append(opts, config.WithDefaultRegion(defaultRegion))
+ }
+
+ accessKey, secretKey := props[S3AccessKeyID], props[S3SecretAccessKey]
Review Comment:
```suggestion
accessKey, secretAccessKey := props[S3AccessKeyID], props[S3SecretAccessKey]
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] nastra commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1331897950
##########
manifest.go:
##########
@@ -0,0 +1,655 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package iceberg
+
+import (
+ "io"
+ "sync"
+
+ iceio "github.com/apache/iceberg-go/io"
+
+ "github.com/hamba/avro/v2/ocf"
+)
+
+// ManifestContent indicates the type of data inside of the files
+// described by a manifest. This will indicate whether the data files
+// contain active data or deleted rows.
+type ManifestContent int32
+
+const (
+ ManifestContentData ManifestContent = 0
+ ManifestContentDeletes ManifestContent = 1
+)
+
+type fieldSummary struct {
+ ContainsNull bool `avro:"contains_null"`
+ ContainsNaN *bool `avro:"contains_nan"`
+ LowerBound *[]byte `avro:"lower_bound"`
+ UpperBound *[]byte `avro:"upper_bound"`
+}
+
+type manifestFileV1 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
Review Comment:
so it seems that the Spec itself for v1 should have `added_snapshot_id` as `optional`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] nastra commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1334934741
##########
manifest.go:
##########
@@ -0,0 +1,655 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package iceberg
+
+import (
+ "io"
+ "sync"
+
+ iceio "github.com/apache/iceberg-go/io"
+
+ "github.com/hamba/avro/v2/ocf"
+)
+
+// ManifestContent indicates the type of data inside of the files
+// described by a manifest. This will indicate whether the data files
+// contain active data or deleted rows.
+type ManifestContent int32
+
+const (
+ ManifestContentData ManifestContent = 0
+ ManifestContentDeletes ManifestContent = 1
+)
+
+type fieldSummary struct {
+ ContainsNull bool `avro:"contains_null"`
+ ContainsNaN *bool `avro:"contains_nan"`
+ LowerBound *[]byte `avro:"lower_bound"`
+ UpperBound *[]byte `avro:"upper_bound"`
+}
+
+type manifestFileV1 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ AddedSnapshotID *int64 `avro:"added_snapshot_id"`
+ AddedFilesCount *int32 `avro:"added_data_files_count"`
+ ExistingFilesCount *int32 `avro:"existing_data_files_count"`
+ DeletedFilesCount *int32 `avro:"deleted_data_files_count"`
+ AddedRowsCount *int64 `avro:"added_rows_count"`
+ ExistingRowsCount *int64 `avro:"existing_rows_count"`
+ DeletedRowsCount *int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV1) Version() int { return 1 }
+func (m *manifestFileV1) FilePath() string { return m.Path }
+func (m *manifestFileV1) Length() int64 { return m.Len }
+func (m *manifestFileV1) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV1) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV1) SnapshotID() int64 {
+ if m.AddedSnapshotID == nil {
+ return 0
+ }
+ return *m.AddedSnapshotID
+}
+
+func (m *manifestFileV1) AddedDataFiles() int32 {
+ if m.AddedFilesCount == nil {
+ return 0
+ }
+ return *m.AddedFilesCount
+}
+
+func (m *manifestFileV1) ExistingDataFiles() int32 {
+ if m.ExistingFilesCount == nil {
+ return 0
+ }
+ return *m.ExistingFilesCount
+}
+
+func (m *manifestFileV1) DeletedDataFiles() int32 {
+ if m.DeletedFilesCount == nil {
+ return 0
+ }
+ return *m.DeletedFilesCount
+}
+
+func (m *manifestFileV1) AddedRows() int64 {
+ if m.AddedRowsCount == nil {
+ return 0
+ }
+ return *m.AddedRowsCount
+}
+
+func (m *manifestFileV1) ExistingRows() int64 {
+ if m.ExistingRowsCount == nil {
+ return 0
+ }
+ return *m.ExistingRowsCount
+}
+
+func (m *manifestFileV1) DeletedRows() int64 {
+ if m.DeletedRowsCount == nil {
+ return 0
+ }
+ return *m.DeletedRowsCount
+}
+
+func (m *manifestFileV1) HasAddedFiles() bool {
+ return m.AddedFilesCount == nil || *m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV1) HasExistingFiles() bool {
+ return m.ExistingFilesCount == nil || *m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV1) SequenceNum() int64 { return 0 }
+func (m *manifestFileV1) MinSequenceNum() int64 { return 0 }
+func (m *manifestFileV1) Metadata() []byte { return m.KeyMetadata }
+func (m *manifestFileV1) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV1) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+type manifestFileV2 struct {
+ Path string `avro:"manifest_path"`
+ Len int64 `avro:"manifest_length"`
+ PartitionSpecID int32 `avro:"partition_spec_id"`
+ Content ManifestContent `avro:"content"`
+ SeqNumber int64 `avro:"sequence_number"`
+ MinSeqNumber int64 `avro:"min_sequence_number"`
+ AddedSnapshotID int64 `avro:"added_snapshot_id"`
+ AddedFilesCount int32 `avro:"added_files_count"`
+ ExistingFilesCount int32 `avro:"existing_files_count"`
+ DeletedFilesCount int32 `avro:"deleted_files_count"`
+ AddedRowsCount int64 `avro:"added_rows_count"`
+ ExistingRowsCount int64 `avro:"existing_rows_count"`
+ DeletedRowsCount int64 `avro:"deleted_rows_count"`
+ Partitions *[]fieldSummary `avro:"partitions"`
+ KeyMetadata []byte `avro:"key_metadata"`
+}
+
+func (*manifestFileV2) Version() int { return 2 }
+
+func (m *manifestFileV2) FilePath() string { return m.Path }
+func (m *manifestFileV2) Length() int64 { return m.Len }
+func (m *manifestFileV2) PartitionID() int32 { return m.PartitionSpecID }
+func (m *manifestFileV2) ManifestContent() ManifestContent { return m.Content }
+func (m *manifestFileV2) SnapshotID() int64 {
+ return m.AddedSnapshotID
+}
+
+func (m *manifestFileV2) AddedDataFiles() int32 {
+ return m.AddedFilesCount
+}
+
+func (m *manifestFileV2) ExistingDataFiles() int32 {
+ return m.ExistingFilesCount
+}
+
+func (m *manifestFileV2) DeletedDataFiles() int32 {
+ return m.DeletedFilesCount
+}
+
+func (m *manifestFileV2) AddedRows() int64 {
+ return m.AddedRowsCount
+}
+
+func (m *manifestFileV2) ExistingRows() int64 {
+ return m.ExistingRowsCount
+}
+
+func (m *manifestFileV2) DeletedRows() int64 {
+ return m.DeletedRowsCount
+}
+
+func (m *manifestFileV2) SequenceNum() int64 { return m.SeqNumber }
+func (m *manifestFileV2) MinSequenceNum() int64 { return m.MinSeqNumber }
+func (m *manifestFileV2) Metadata() []byte { return m.KeyMetadata }
+
+func (m *manifestFileV2) PartitionList() []fieldSummary {
+ if m.Partitions == nil {
+ return nil
+ }
+ return *m.Partitions
+}
+
+func (m *manifestFileV2) HasAddedFiles() bool {
+ return m.AddedFilesCount > 0
+}
+
+func (m *manifestFileV2) HasExistingFiles() bool {
+ return m.ExistingFilesCount > 0
+}
+
+func (m *manifestFileV2) FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ return fetchManifestEntries(m, fs, discardDeleted)
+}
+
+func fetchManifestEntries(m ManifestFile, fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) {
+ f, err := fs.Open(m.FilePath())
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+
+ dec, err := ocf.NewDecoder(f)
+ if err != nil {
+ return nil, err
+ }
+
+ metadata := dec.Metadata()
+ isVer1 := true
+ if string(metadata["format-version"]) == "2" {
+ isVer1 = false
+ }
+
+ results := make([]ManifestEntry, 0)
+ for dec.HasNext() {
+ var tmp ManifestEntry
+ if isVer1 {
+ tmp = &manifestEntryV1{}
+ } else {
+ tmp = &manifestEntryV2{}
+ }
+
+ if err := dec.Decode(tmp); err != nil {
+ return nil, err
+ }
+
+ if !discardDeleted || tmp.Status() != EntryStatusDELETED {
+ tmp.inheritSeqNum(m)
+ results = append(results, tmp)
+ }
+ }
+
+ return results, dec.Error()
+}
+
+// ManifestFile is the interface which covers both V1 and V2 manifest files.
+type ManifestFile interface {
+ // Version returns the version number of this manifest file.
+ // It should be 1 or 2.
+ Version() int
+ // FilePath is the location URI of this manifest file.
+ FilePath() string
+ // Length is the length in bytes of the manifest file.
+ Length() int64
+ // PartitionID is the ID of the partition spec used to write
+ // this manifest. It must be listed in the table metadata
+ // partition-specs.
+ PartitionID() int32
+ // ManifestContent is the type of files tracked by this manifest,
+ // either data or delete files. All v1 manifests track data files.
+ ManifestContent() ManifestContent
+ // SnapshotID is the ID of the snapshot where this manifest file
+ // was added.
+ SnapshotID() int64
+ // AddedDataFiles returns the number of entries in the manifest that
+ // have the status of EntryStatusADDED.
+ AddedDataFiles() int32
+ // ExistingDataFiles returns the number of entries in the manifest
+ // which have the status of EntryStatusEXISTING.
+ ExistingDataFiles() int32
+ // DeletedDataFiles returns the number of entries in the manifest
+ // which have the status of EntryStatusDELETED.
+ DeletedDataFiles() int32
+ // AddedRows returns the number of rows in all files of the manifest
+ // that have status EntryStatusADDED.
+ AddedRows() int64
+ // ExistingRows returns the number of rows in all files of the manifest
+ // which have status EntryStatusEXISTING.
+ ExistingRows() int64
+ // DeletedRows returns the number of rows in all files of the manifest
+ // which have status EntryStatusDELETED.
+ DeletedRows() int64
+ // SequenceNum returns the sequence number when this manifest was
+ // added to the table. Will be 0 for v1 manifest lists.
+ SequenceNum() int64
+ // MinSequenceNum is the minimum data sequence number of all live data
+ // or delete files in the manifest. Will be 0 for v1 manifest lists.
+ MinSequenceNum() int64
+ // Metadata returns implementation-specific key metadata for encryption
+ // if it exists in the manifest list.
+ Metadata() []byte
+ // PartitionList returns a list of field summaries for each partition
+ // field in the spec. Each field in the list corresponds to a field in
+ // the manifest file's partition spec.
+ PartitionList() []fieldSummary
+
+ // HasAddedFiles returns true if AddedDataFiles > 0 or if it was null.
+ HasAddedFiles() bool
+ // HasExistingFiles returns true if ExistingDataFiles > 0 or if it was null.
+ HasExistingFiles() bool
+ // FetchEntries reads the manifest list file to fetch the list of
+ // manifest entries using the provided file system IO interface.
+ // If discardDeleted is true, entries for files containing deleted rows
+ // will be skipped.
+ FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error)
+}
+
+// ReadManifestList reads in an avro manifest list file and returns a slice
+// of manifest files or an error if one is encountered.
+func ReadManifestList(in io.Reader) ([]ManifestFile, error) {
+ dec, err := ocf.NewDecoder(in)
+ if err != nil {
+ return nil, err
+ }
+
+ out := make([]ManifestFile, 0)
+
+ for dec.HasNext() {
+ var file ManifestFile
+ if string(dec.Metadata()["format-version"]) == "2" {
+ file = &manifestFileV2{}
+ } else {
+ file = &manifestFileV1{}
+ }
+
+ if err := dec.Decode(file); err != nil {
+ return nil, err
+ }
+ out = append(out, file)
+ }
+
+ return out, dec.Error()
+}
+
+// ManifestEntryStatus defines constants for the entry status of
+// existing, added or deleted.
+type ManifestEntryStatus int8
+
+const (
+ EntryStatusEXISTING ManifestEntryStatus = 0
+ EntryStatusADDED ManifestEntryStatus = 1
+ EntryStatusDELETED ManifestEntryStatus = 2
+)
+
+// ManifestEntryContent defines constants for the type of file contents
+// in the file entries. Data, Position based deletes and equality based
+// deletes.
+type ManifestEntryContent int8
+
+const (
+ EntryContentData ManifestEntryContent = 0
+ EntryContentPosDeletes ManifestEntryContent = 1
+ EntryContentEqDeletes ManifestEntryContent = 2
+)
+
+// FileFormat defines constants for the format of data files.
+type FileFormat string
+
+const (
+ AvroFile FileFormat = "AVRO"
+ OrcFile FileFormat = "ORC"
+ ParquetFile FileFormat = "PARQUET"
+)
+
+type colMap[K, V any] struct {
+ Key K `avro:"key"`
+ Value V `avro:"value"`
+}
+
+func avroColMapToMap[K comparable, V any](c *[]colMap[K, V]) map[K]V {
+ if c == nil {
+ return nil
+ }
+
+ out := make(map[K]V)
+ for _, data := range *c {
+ out[data.Key] = data.Value
+ }
+ return out
+}
+
+type dataFile struct {
+ Content ManifestEntryContent `avro:"content"`
Review Comment:
great, thanks for confirming
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg-go] zeroshade commented on a diff in pull request #3: feat(manifests): Adding implementation of manifest files
Posted by "zeroshade (via GitHub)" <gi...@apache.org>.
zeroshade commented on code in PR #3:
URL: https://github.com/apache/iceberg-go/pull/3#discussion_r1334837214
##########
internal/avro_schemas.go:
##########
@@ -0,0 +1,568 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package internal
+
+import "github.com/hamba/avro/v2"
+
+const (
+ ManifestListV1Key = "manifest-list-v1"
+ ManifestListV2Key = "manifest-list-v2"
+ ManifestEntryV1Key = "manifest-entry-v1"
+ ManifestEntryV2Key = "manifest-entry-v2"
+)
+
+var (
+ AvroSchemaCache avro.SchemaCache
+)
+
+func init() {
+ AvroSchemaCache.Add(ManifestListV1Key, avro.MustParse(`{
+ "type": "record",
+ "name": "manifest_file",
+ "fields": [
+ {"name": "manifest_path", "type": "string", "doc": "Location URI with FS scheme", "field-id": 500},
+ {"name": "manifest_length", "type": "long", "doc": "Total file size in bytes", "field-id": 501},
+ {"name": "partition_spec_id", "type": "int", "doc": "Spec ID used to write", "field-id": 502},
+ {
+ "name": "added_snapshot_id",
+ "type": "long",
Review Comment:
fixed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org