You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@iceberg.apache.org by "gty404 (via GitHub)" <gi...@apache.org> on 2025/05/27 06:46:06 UTC
[PR] feat: basic table scan planning [iceberg-cpp]
gty404 opened a new pull request, #112:
URL: https://github.com/apache/iceberg-cpp/pull/112
Introducing basic scan table data interface
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "gty404 (via GitHub)" <gi...@apache.org>.
gty404 commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2177789588
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+/// \brief Represents a task to scan a table or a portion of it.
+class ICEBERG_EXPORT ScanTask {
+ public:
+ virtual ~ScanTask() = default;
+
+ /// \brief The number of bytes that should be read by this scan task.
+ virtual int64_t size_bytes() const = 0;
+
+ /// \brief The number of files that should be read by this scan task.
+ virtual int32_t files_count() const = 0;
+
+ /// \brief The number of rows that should be read by this scan task.
+ virtual int64_t estimated_row_count() const = 0;
+};
+
+/// \brief Represents a task to scan a portion of a data file.
+class ICEBERG_EXPORT FileScanTask : public ScanTask {
+ public:
+ FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files, int64_t start,
+ int64_t length, std::shared_ptr<Expression> residual);
+
+ /// \brief The data file that should be read by this scan task.
+ const std::shared_ptr<DataFile>& data_file() const;
+
+ /// \brief The delete files that should be read by this scan task.
+ const std::vector<std::shared_ptr<DataFile>>& delete_files() const;
+
+ /// \brief The byte offset in the data file where the scan should start.
+ int64_t start() const;
+
+ /// \brief The length in bytes to scan from the start offset.
+ int64_t length() const;
+
+ /// \brief The residual expression to apply after scanning the data file.
+ const std::shared_ptr<Expression>& residual() const;
+
+ int64_t size_bytes() const override;
+ int32_t files_count() const override;
+ int64_t estimated_row_count() const override;
+
+ private:
+ /// \brief Data file metadata.
+ std::shared_ptr<DataFile> data_file_;
+ /// \brief Delete files metadata.
+ std::vector<std::shared_ptr<DataFile>> delete_files_;
+ /// \brief Start byte offset.
+ int64_t start_;
+ /// \brief Length in bytes to scan.
+ int64_t length_;
+ /// \brief Residual expression to apply.
+ std::shared_ptr<Expression> residual_;
+};
+
+/// \brief Scan context holding snapshot and scan-specific metadata.
+struct TableScanContext {
+ /// \brief Table metadata.
+ std::shared_ptr<TableMetadata> table_metadata;
+ /// \brief Snapshot to scan.
+ std::shared_ptr<Snapshot> snapshot;
+ /// \brief Projected schema.
+ std::shared_ptr<Schema> projected_schema;
+ /// \brief Filter expression to apply.
+ std::shared_ptr<Expression> filter;
+ /// \brief Whether the scan is case-sensitive.
+ bool case_sensitive = false;
+ /// \brief Additional options for the scan.
+ std::unordered_map<std::string, std::string> options;
+ /// \brief Optional limit on the number of rows to scan.
+ std::optional<int64_t> limit;
+};
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table The table to scan.
+ /// \param table_metadata The metadata of the table to scan.
+ explicit TableScanBuilder(const Table& table,
+ std::shared_ptr<TableMetadata> table_metadata);
+
+ /// \brief Sets the snapshot ID to scan.
+ /// \param snapshot_id The ID of the snapshot.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSnapshotId(int64_t snapshot_id);
+
+ /// \brief Selects columns to include in the scan.
+ /// \param column_names A list of column names. If empty, all columns will be selected.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithColumnNames(std::vector<std::string> column_names);
+
+ /// \brief Sets the schema to use for the scan.
+ /// \param schema The schema to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSchema(std::shared_ptr<Schema> schema);
Review Comment:
This is used to specify the projected schema without specifying the column name, and I have modified it to WithProjectedSchema.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "wgtmac (via GitHub)" <gi...@apache.org>.
wgtmac commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2184733959
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+/// \brief An abstract scan task.
+class ICEBERG_EXPORT ScanTask {
+ public:
+ virtual ~ScanTask() = default;
+
+ /// \brief The number of bytes that should be read by this scan task.
+ virtual int64_t SizeBytes() const = 0;
+
+ /// \brief The number of files that should be read by this scan task.
+ virtual int32_t FilesCount() const = 0;
+
+ /// \brief The number of rows that should be read by this scan task.
+ virtual int64_t EstimatedRowCount() const = 0;
+};
+
+/// \brief Task representing a data file and its corresponding delete files.
+class ICEBERG_EXPORT FileScanTask : public ScanTask {
+ public:
+ FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files,
+ std::shared_ptr<Expression> residual);
+
+ /// \brief The data file that should be read by this scan task.
+ const std::shared_ptr<DataFile>& data_file() const;
+
+ /// \brief The delete files that should be read by this scan task.
+ const std::vector<std::shared_ptr<DataFile>>& delete_files() const;
+
+ /// \brief The residual expression to apply after scanning the data file.
+ const std::shared_ptr<Expression>& residual() const;
+
+ int64_t SizeBytes() const override;
+ int32_t FilesCount() const override;
+ int64_t EstimatedRowCount() const override;
+
+ private:
+ /// \brief Data file metadata.
+ std::shared_ptr<DataFile> data_file_;
+ /// \brief Delete files metadata.
+ std::vector<std::shared_ptr<DataFile>> delete_files_;
+ /// \brief Residual expression to apply.
+ std::shared_ptr<Expression> residual_;
+};
+
+/// \brief Scan context holding snapshot and scan-specific metadata.
+struct TableScanContext {
+ /// \brief Table metadata.
+ std::shared_ptr<TableMetadata> table_metadata;
+ /// \brief Snapshot to scan.
+ std::shared_ptr<Snapshot> snapshot;
+ /// \brief Projected schema.
+ std::shared_ptr<Schema> projected_schema;
+ /// \brief Filter expression to apply.
+ std::shared_ptr<Expression> filter;
+ /// \brief Whether the scan is case-sensitive.
+ bool case_sensitive = false;
+ /// \brief Additional options for the scan.
+ std::unordered_map<std::string, std::string> options;
+ /// \brief Optional limit on the number of rows to scan.
+ std::optional<int64_t> limit;
+};
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table_metadata The metadata of the table to scan.
+ /// \param file_io The FileIO instance for reading manifests and data files.
+ explicit TableScanBuilder(std::shared_ptr<TableMetadata> table_metadata,
+ std::shared_ptr<FileIO> file_io);
+
+ /// \brief Sets the snapshot ID to scan.
+ /// \param snapshot_id The ID of the snapshot.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSnapshotId(int64_t snapshot_id);
+
+ /// \brief Selects columns to include in the scan.
+ /// \param column_names A list of column names. If empty, all columns will be selected.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithColumnNames(std::vector<std::string> column_names);
+
+ /// \brief Sets the schema to use for the scan.
+ /// \param schema The schema to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithProjectedSchema(std::shared_ptr<Schema> schema);
+
+ /// \brief Applies a filter expression to the scan.
+ /// \param filter Filter expression to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithFilter(std::shared_ptr<Expression> filter);
+
+ /// \brief Sets whether the scan should be case-sensitive.
+ /// \param case_sensitive Whether the scan is case-sensitive.
+ /// /return Reference to the builder.
+ TableScanBuilder& WithCaseSensitive(bool case_sensitive);
+
+ /// \brief Sets an option for the scan.
+ /// \param property The name of the option.
+ /// \param value The value of the option.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithOption(std::string property, std::string value);
+
+ /// \brief Sets an optional limit on the number of rows to scan.
+ /// \param limit Optional limit on the number of rows.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithLimit(std::optional<int64_t> limit);
+
+ /// \brief Builds and returns a TableScan instance.
+ /// \return A Result containing the TableScan or an error.
+ Result<std::unique_ptr<TableScan>> Build();
+
+ private:
+ /// \brief the file I/O instance for reading manifests and data files.
+ std::shared_ptr<FileIO> file_io_;
+ /// \brief column names to project in the scan.
+ std::vector<std::string> column_names_;
+ /// \brief snapshot ID to scan, if specified.
+ std::optional<int64_t> snapshot_id_;
+ /// \brief Context for the scan, including snapshot, schema, and filter.
+ TableScanContext context_;
+};
+
+/// \brief Represents a configured scan operation on a table.
+class ICEBERG_EXPORT TableScan {
+ public:
+ virtual ~TableScan() = default;
+
+ /// \brief Constructs a TableScan with the given context and file I/O.
+ /// \param context Scan context including snapshot, schema, and filter.
+ /// \param file_io File I/O instance for reading manifests and data files.
+ TableScan(TableScanContext context, std::shared_ptr<FileIO> file_io);
+
+ /// \brief Returns the snapshot being scanned.
+ /// \return A shared pointer to the snapshot.
+ const std::shared_ptr<Snapshot>& snapshot() const;
+
+ /// \brief Returns the projected schema for the scan.
+ /// \return A shared pointer to the projected schema.
+ const std::shared_ptr<Schema>& projection() const;
+
+ /// \brief Returns the scan context.
+ /// \return A reference to the TableScanContext.
+ const TableScanContext& context() const;
+
+ /// \brief Returns the file I/O instance used for reading manifests and data files.
+ /// \return A shared pointer to the FileIO instance.
+ const std::shared_ptr<FileIO>& io() const;
+
+ /// \brief Plans the scan tasks by resolving manifests and data files.
+ /// \return A Result containing scan tasks or an error.
+ virtual Result<std::vector<std::shared_ptr<FileScanTask>>> PlanFiles() const = 0;
+
+ protected:
+ /// \brief context for the scan, including snapshot, schema, and filter.
+ const TableScanContext context_;
+ /// \brief File I/O instance for reading manifests and data files.
+ std::shared_ptr<FileIO> file_io_;
+};
+
+/// \brief A scan that reads data files and applies delete files to filter rows.
+class ICEBERG_EXPORT DataScan : public TableScan {
Review Comment:
```suggestion
class ICEBERG_EXPORT DataTableScan : public TableScan {
```
I think `DataScan` is ambiguous and people may assume that it will return rows of data instead of the file list. `DataTable` is better to differentiate from `MetadataTable`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "yingcai-cy (via GitHub)" <gi...@apache.org>.
yingcai-cy commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2128307241
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+TableScanBuilder::TableScanBuilder(const Table& table) : table_(table) {}
+
+TableScanBuilder& TableScanBuilder::WithColumnNames(
+ const std::vector<std::string>& column_names) {
+ column_names_ = column_names;
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithSnapshotId(int64_t snapshot_id) {
+ snapshot_id_ = snapshot_id;
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithFilter(
+ const std::shared_ptr<Expression>& filter) {
+ filter_ = filter;
+ return *this;
+}
+
+Result<std::unique_ptr<TableScan>> TableScanBuilder::Build() {
+ std::shared_ptr<Snapshot> snapshot;
+ if (snapshot_id_) {
+ ICEBERG_ASSIGN_OR_RAISE(snapshot, table_.snapshot(*snapshot_id_));
+ } else {
+ snapshot = table_.current_snapshot();
Review Comment:
Can current_snapshot() return nullptr?
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table Reference to the table to scan.
+ explicit TableScanBuilder(const Table& table);
+
+ /// \brief Sets the snapshot ID to scan.
+ /// \param snapshot_id The ID of the snapshot.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSnapshotId(int64_t snapshot_id);
+
+ /// \brief Selects columns to include in the scan.
+ /// Defaults to none which means select all columns
+ /// \param column_names A list of column names.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithColumnNames(const std::vector<std::string>& column_names);
Review Comment:
```suggestion
TableScanBuilder& WithColumnNames(std::vector<std::string> column_names);
```
Pass by value and use move in implementation.
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+TableScanBuilder::TableScanBuilder(const Table& table) : table_(table) {}
+
+TableScanBuilder& TableScanBuilder::WithColumnNames(
+ const std::vector<std::string>& column_names) {
+ column_names_ = column_names;
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithSnapshotId(int64_t snapshot_id) {
+ snapshot_id_ = snapshot_id;
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithFilter(
+ const std::shared_ptr<Expression>& filter) {
+ filter_ = filter;
+ return *this;
+}
+
+Result<std::unique_ptr<TableScan>> TableScanBuilder::Build() {
+ std::shared_ptr<Snapshot> snapshot;
+ if (snapshot_id_) {
+ ICEBERG_ASSIGN_OR_RAISE(snapshot, table_.snapshot(*snapshot_id_));
+ } else {
+ snapshot = table_.current_snapshot();
+ }
+
+ std::shared_ptr<Schema> schema;
+ if (snapshot->schema_id) {
+ const auto& schemas = table_.schemas();
+ if (auto it = schemas.find(*snapshot->schema_id); it != schemas.end()) {
+ schema = it->second;
+ } else {
+ return InvalidData("Schema {} in snapshot {} is not found", *snapshot->schema_id,
Review Comment:
nit:I think use InvalidArgument error code is more accurate.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2182149699
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,294 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+namespace {
+/// \brief Use indexed data structures for efficient lookups
+class DeleteFileIndex {
+ public:
+ /// \brief Build the index from a list of manifest entries.
+ explicit DeleteFileIndex(const std::vector<std::unique_ptr<ManifestEntry>>& entries) {
+ for (const auto& entry : entries) {
+ const int64_t seq_num =
+ entry->sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ sequence_index.emplace(seq_num, entry.get());
+ }
+ }
+
+ /// \brief Find delete files that match the sequence number of a data entry.
+ std::vector<ManifestEntry*> FindRelevantEntries(const ManifestEntry& data_entry) const {
+ std::vector<ManifestEntry*> relevant_deletes;
+
+ // Use lower_bound for efficient range search
+ auto data_sequence_number =
+ data_entry.sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ for (auto it = sequence_index.lower_bound(data_sequence_number);
+ it != sequence_index.end(); ++it) {
+ // Additional filtering logic here
+ relevant_deletes.push_back(it->second);
+ }
+
+ return relevant_deletes;
+ }
+
+ private:
+ /// \brief Index by sequence number for quick filtering
+ std::multimap<int64_t, ManifestEntry*> sequence_index;
+};
+
+/// \brief Get matched delete files for a given data entry.
+std::vector<std::shared_ptr<DataFile>> GetMatchedDeletes(
+ const ManifestEntry& data_entry, const DeleteFileIndex& delete_file_index) {
+ const auto relevant_entries = delete_file_index.FindRelevantEntries(data_entry);
+ std::vector<std::shared_ptr<DataFile>> matched_deletes;
+ if (relevant_entries.empty()) {
+ return matched_deletes;
+ }
+
+ matched_deletes.reserve(relevant_entries.size());
+ for (const auto& delete_entry : relevant_entries) {
+ // TODO(gty404): check if the delete entry contains the data entry's file path
+ matched_deletes.emplace_back(delete_entry->data_file);
+ }
+ return matched_deletes;
+}
+} // namespace
+
+// implement FileScanTask
+FileScanTask::FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files,
+ int64_t start, int64_t length,
+ std::shared_ptr<Expression> residual)
+ : data_file_(std::move(file)),
+ delete_files_(std::move(delete_files)),
+ start_(start),
+ length_(length),
+ residual_(std::move(residual)) {}
Review Comment:
It looks like this `FileScanTask` is inspired by PyIceberg, while I think it might be better to follow Java: https://github.com/apache/iceberg/blob/main/api/src/main/java/org/apache/iceberg/FileScanTask.java
The name `FileScanTask` implies to me, that it will read a full file, and then the `start` and `end` do not make sense. This is in the case you want to split up the work by row-group, rather than file.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "gty404 (via GitHub)" <gi...@apache.org>.
gty404 commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2174097517
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+// implement FileScanTask
+FileScanTask::FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files,
+ int64_t start, int64_t length,
+ std::shared_ptr<Expression> residual)
+ : data_file_(std::move(file)),
+ delete_files_(std::move(delete_files)),
+ start_(start),
+ length_(length),
+ residual_(std::move(residual)) {}
+
+const std::shared_ptr<DataFile>& FileScanTask::data_file() const { return data_file_; }
+
+const std::vector<std::shared_ptr<DataFile>>& FileScanTask::delete_files() const {
+ return delete_files_;
+}
+
+int64_t FileScanTask::start() const { return start_; }
+
+int64_t FileScanTask::length() const { return length_; }
+
+int64_t FileScanTask::size_bytes() const {
+ int64_t sizeInBytes = length_;
+ std::ranges::for_each(delete_files_, [&sizeInBytes](const auto& delete_file) {
+ sizeInBytes += delete_file->file_size_in_bytes;
+ });
+ return sizeInBytes;
+}
+
+int32_t FileScanTask::files_count() const {
+ return static_cast<int32_t>(delete_files_.size() + 1);
+}
+
+int64_t FileScanTask::estimated_row_count() const {
+ if (data_file_->file_size_in_bytes == 0) {
+ return 0;
+ }
+ const double scannedFileFraction =
+ static_cast<double>(length_) / data_file_->file_size_in_bytes;
+ return static_cast<int64_t>(scannedFileFraction * data_file_->record_count);
+}
+
+const std::shared_ptr<Expression>& FileScanTask::residual() const { return residual_; }
+
+TableScanBuilder::TableScanBuilder(const Table& table,
+ std::shared_ptr<TableMetadata> table_metadata)
+ : table_(table) {
+ context_.table_metadata = std::move(table_metadata);
+}
+
+TableScanBuilder& TableScanBuilder::WithColumnNames(
+ std::vector<std::string> column_names) {
+ column_names_.reserve(column_names.size());
+ column_names_ = std::move(column_names);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithSchema(std::shared_ptr<Schema> schema) {
+ context_.projected_schema = std::move(schema);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithSnapshotId(int64_t snapshot_id) {
+ snapshot_id_ = snapshot_id;
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithFilter(std::shared_ptr<Expression> filter) {
+ context_.filter = std::move(filter);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithCaseSensitive(bool case_sensitive) {
+ context_.case_sensitive = case_sensitive;
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithOption(std::string property, std::string value) {
+ context_.options[std::move(property)] = std::move(value);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithLimit(std::optional<int64_t> limit) {
+ context_.limit = limit;
+ return *this;
+}
+
+Result<std::unique_ptr<TableScan>> TableScanBuilder::Build() {
+ if (snapshot_id_) {
+ ICEBERG_ASSIGN_OR_RAISE(context_.snapshot, table_.snapshot(*snapshot_id_));
+ } else {
+ context_.snapshot = table_.current_snapshot();
+ }
+ if (context_.snapshot == nullptr) {
+ return InvalidArgument("No snapshot found for table {}", table_.name());
+ }
+
+ if (!context_.projected_schema) {
+ std::shared_ptr<Schema> schema;
+ const auto& snapshot = context_.snapshot;
+ if (snapshot->schema_id) {
+ const auto& schemas = table_.schemas();
+ if (const auto it = schemas.find(*snapshot->schema_id); it != schemas.end()) {
+ schema = it->second;
+ } else {
+ return InvalidArgument("Schema {} in snapshot {} is not found",
+ *snapshot->schema_id, snapshot->snapshot_id);
+ }
+ } else {
+ schema = table_.schema();
+ }
+
+ // TODO(gty404): collect touched columns from filter expression
+ std::vector<SchemaField> projected_fields;
+ projected_fields.reserve(column_names_.size());
Review Comment:
If empty, all columns will be selected
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "gty404 (via GitHub)" <gi...@apache.org>.
gty404 commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2183983587
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+namespace {
+/// \brief Use indexed data structures for efficient lookups
+class DeleteFileIndex {
+ public:
+ /// \brief Build the index from a list of manifest entries.
+ explicit DeleteFileIndex(const std::vector<std::unique_ptr<ManifestEntry>>& entries) {
+ for (const auto& entry : entries) {
+ const int64_t seq_num =
+ entry->sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ sequence_index.emplace(seq_num, entry.get());
+ }
+ }
+
+ /// \brief Find delete files that match the sequence number of a data entry.
+ std::vector<ManifestEntry*> FindRelevantEntries(const ManifestEntry& data_entry) const {
+ std::vector<ManifestEntry*> relevant_deletes;
+
+ // Use lower_bound for efficient range search
+ auto data_sequence_number =
+ data_entry.sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ for (auto it = sequence_index.lower_bound(data_sequence_number);
+ it != sequence_index.end(); ++it) {
+ // Additional filtering logic here
+ relevant_deletes.push_back(it->second);
+ }
+
+ return relevant_deletes;
+ }
+
+ private:
+ /// \brief Index by sequence number for quick filtering
+ std::multimap<int64_t, ManifestEntry*> sequence_index;
Review Comment:
It is for the convenience of searching by sequence as mentioned earlier.
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+namespace {
+/// \brief Use indexed data structures for efficient lookups
+class DeleteFileIndex {
+ public:
+ /// \brief Build the index from a list of manifest entries.
+ explicit DeleteFileIndex(const std::vector<std::unique_ptr<ManifestEntry>>& entries) {
+ for (const auto& entry : entries) {
+ const int64_t seq_num =
+ entry->sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ sequence_index.emplace(seq_num, entry.get());
+ }
+ }
+
+ /// \brief Find delete files that match the sequence number of a data entry.
+ std::vector<ManifestEntry*> FindRelevantEntries(const ManifestEntry& data_entry) const {
+ std::vector<ManifestEntry*> relevant_deletes;
+
+ // Use lower_bound for efficient range search
+ auto data_sequence_number =
+ data_entry.sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ for (auto it = sequence_index.lower_bound(data_sequence_number);
+ it != sequence_index.end(); ++it) {
Review Comment:
Yes, the meaning here is to find all DeleteFiles corresponding to this DataFile. Only those with a sequence number higher than the DataFile need to be read.
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+namespace {
+/// \brief Use indexed data structures for efficient lookups
+class DeleteFileIndex {
+ public:
+ /// \brief Build the index from a list of manifest entries.
+ explicit DeleteFileIndex(const std::vector<std::unique_ptr<ManifestEntry>>& entries) {
+ for (const auto& entry : entries) {
+ const int64_t seq_num =
+ entry->sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ sequence_index.emplace(seq_num, entry.get());
+ }
+ }
+
+ /// \brief Find delete files that match the sequence number of a data entry.
+ std::vector<ManifestEntry*> FindRelevantEntries(const ManifestEntry& data_entry) const {
+ std::vector<ManifestEntry*> relevant_deletes;
+
+ // Use lower_bound for efficient range search
+ auto data_sequence_number =
+ data_entry.sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ for (auto it = sequence_index.lower_bound(data_sequence_number);
+ it != sequence_index.end(); ++it) {
+ // Additional filtering logic here
+ relevant_deletes.push_back(it->second);
+ }
+
+ return relevant_deletes;
+ }
+
+ private:
+ /// \brief Index by sequence number for quick filtering
+ std::multimap<int64_t, ManifestEntry*> sequence_index;
+};
+
+/// \brief Get matched delete files for a given data entry.
+std::vector<std::shared_ptr<DataFile>> GetMatchedDeletes(
+ const ManifestEntry& data_entry, const DeleteFileIndex& delete_file_index) {
+ const auto relevant_entries = delete_file_index.FindRelevantEntries(data_entry);
+ std::vector<std::shared_ptr<DataFile>> matched_deletes;
+ if (relevant_entries.empty()) {
+ return matched_deletes;
+ }
+
+ matched_deletes.reserve(relevant_entries.size());
+ for (const auto& delete_entry : relevant_entries) {
+ // TODO(gty404): check if the delete entry contains the data entry's file path
+ matched_deletes.emplace_back(delete_entry->data_file);
+ }
+ return matched_deletes;
+}
+} // namespace
+
+// implement FileScanTask
+FileScanTask::FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files,
+ std::shared_ptr<Expression> residual)
+ : data_file_(std::move(file)),
+ delete_files_(std::move(delete_files)),
+ residual_(std::move(residual)) {}
+
+const std::shared_ptr<DataFile>& FileScanTask::data_file() const { return data_file_; }
+
+const std::vector<std::shared_ptr<DataFile>>& FileScanTask::delete_files() const {
+ return delete_files_;
+}
+
+int64_t FileScanTask::SizeBytes() const {
+ int64_t sizeInBytes = data_file_->file_size_in_bytes;
+ std::ranges::for_each(delete_files_, [&sizeInBytes](const auto& delete_file) {
+ sizeInBytes += delete_file->file_size_in_bytes;
+ });
+ return sizeInBytes;
+}
+
+int32_t FileScanTask::FilesCount() const {
+ return static_cast<int32_t>(delete_files_.size() + 1);
+}
+
+int64_t FileScanTask::EstimatedRowCount() const {
+ if (data_file_->file_size_in_bytes == 0) {
+ return 0;
+ }
+ const auto sizeInBytes = data_file_->file_size_in_bytes;
+ const double scannedFileFraction =
+ static_cast<double>(sizeInBytes) / data_file_->file_size_in_bytes;
+ return static_cast<int64_t>(scannedFileFraction * data_file_->record_count);
+}
+
+const std::shared_ptr<Expression>& FileScanTask::residual() const { return residual_; }
+
+TableScanBuilder::TableScanBuilder(std::shared_ptr<TableMetadata> table_metadata,
+ std::shared_ptr<FileIO> file_io)
+ : file_io_(std::move(file_io)) {
+ context_.table_metadata = std::move(table_metadata);
+}
+
+TableScanBuilder& TableScanBuilder::WithColumnNames(
+ std::vector<std::string> column_names) {
+ column_names_.reserve(column_names.size());
+ column_names_ = std::move(column_names);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithProjectedSchema(std::shared_ptr<Schema> schema) {
+ context_.projected_schema = std::move(schema);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithSnapshotId(int64_t snapshot_id) {
+ snapshot_id_ = snapshot_id;
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithFilter(std::shared_ptr<Expression> filter) {
+ context_.filter = std::move(filter);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithCaseSensitive(bool case_sensitive) {
+ context_.case_sensitive = case_sensitive;
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithOption(std::string property, std::string value) {
+ context_.options[std::move(property)] = std::move(value);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithLimit(std::optional<int64_t> limit) {
+ context_.limit = limit;
+ return *this;
+}
+
+Result<std::unique_ptr<TableScan>> TableScanBuilder::Build() {
+ const auto& table_metadata = context_.table_metadata;
+ auto snapshot_id = snapshot_id_ ? snapshot_id_ : table_metadata->current_snapshot_id;
+ if (!snapshot_id) {
+ return InvalidArgument("No snapshot ID specified for table {}",
+ table_metadata->table_uuid);
+ }
+ auto iter = std::ranges::find_if(table_metadata->snapshots,
+ [&snapshot_id](const auto& snapshot) {
+ return snapshot->snapshot_id == *snapshot_id;
+ });
+ if (iter == table_metadata->snapshots.end() || *iter == nullptr) {
+ return NotFound("Snapshot with ID {} is not found", *snapshot_id);
+ }
+ context_.snapshot = *iter;
+
+ if (!context_.projected_schema) {
+ const auto& snapshot = context_.snapshot;
+ auto schema_id =
+ snapshot->schema_id ? snapshot->schema_id : table_metadata->current_schema_id;
+ if (!schema_id) {
+ return InvalidArgument("No schema ID found in snapshot {} for table {}",
+ snapshot->snapshot_id, table_metadata->table_uuid);
+ }
+
+ const auto& schemas = table_metadata->schemas;
+ const auto it = std::ranges::find_if(schemas, [&schema_id](const auto& schema) {
+ return schema->schema_id() == *schema_id;
+ });
+ if (it == schemas.end()) {
+ return InvalidArgument("Schema {} in snapshot {} is not found",
+ *snapshot->schema_id, snapshot->snapshot_id);
+ }
+ auto schema = *it;
+
+ if (column_names_.empty()) {
+ context_.projected_schema = schema;
+ } else {
+ // TODO(gty404): collect touched columns from filter expression
+ std::vector<SchemaField> projected_fields;
+ projected_fields.reserve(column_names_.size());
+ for (const auto& column_name : column_names_) {
+ // TODO(gty404): support case-insensitive column names
+ auto field_opt = schema->GetFieldByName(column_name);
+ if (!field_opt) {
+ return InvalidArgument("Column {} not found in schema", column_name);
+ }
+ projected_fields.emplace_back(field_opt.value().get());
+ }
+ context_.projected_schema =
+ std::make_shared<Schema>(std::move(projected_fields), schema->schema_id());
+ }
+ }
+
+ return std::make_unique<DataScan>(std::move(context_), file_io_);
+}
+
+TableScan::TableScan(TableScanContext context, std::shared_ptr<FileIO> file_io)
+ : context_(std::move(context)), file_io_(std::move(file_io)) {}
+
+const std::shared_ptr<Snapshot>& TableScan::snapshot() const { return context_.snapshot; }
+
+const std::shared_ptr<Schema>& TableScan::projection() const {
+ return context_.projected_schema;
+}
+
+const TableScanContext& TableScan::context() const { return context_; }
+
+const std::shared_ptr<FileIO>& TableScan::io() const { return file_io_; }
+
+DataScan::DataScan(TableScanContext context, std::shared_ptr<FileIO> file_io)
+ : TableScan(std::move(context), std::move(file_io)) {}
+
+Result<std::vector<std::shared_ptr<FileScanTask>>> DataScan::PlanFiles() const {
+ ICEBERG_ASSIGN_OR_RAISE(auto manifest_list_reader,
+ CreateManifestListReader(context_.snapshot->manifest_list));
+ ICEBERG_ASSIGN_OR_RAISE(auto manifest_files, manifest_list_reader->Files());
+
+ std::vector<std::unique_ptr<ManifestEntry>> data_entries;
+ std::vector<std::unique_ptr<ManifestEntry>> positional_delete_entries;
+ for (const auto& manifest_file : manifest_files) {
+ ICEBERG_ASSIGN_OR_RAISE(auto manifest_reader,
+ CreateManifestReader(manifest_file->manifest_path));
+ ICEBERG_ASSIGN_OR_RAISE(auto manifests, manifest_reader->Entries());
+
+ // TODO(gty404): filter manifests using partition spec and filter expression
+
+ for (auto& manifest_entry : manifests) {
+ const auto& data_file = manifest_entry->data_file;
+ switch (data_file->content) {
+ case DataFile::Content::kData:
+ data_entries.push_back(std::move(manifest_entry));
+ break;
+ case DataFile::Content::kPositionDeletes:
+ // TODO(gty404): check if the sequence number is greater than or equal to the
+ // minimum sequence number of all manifest entries
+ positional_delete_entries.push_back(std::move(manifest_entry));
+ break;
+ case DataFile::Content::kEqualityDeletes:
+ return NotSupported("Equality deletes are not supported in data scan");
+ }
+ }
+ }
+
+ // TODO(gty404): build residual expression from filter
+ std::shared_ptr<Expression> residual;
+ std::vector<std::shared_ptr<FileScanTask>> tasks;
+ DeleteFileIndex delete_file_index(positional_delete_entries);
+ for (const auto& data_entry : data_entries) {
+ auto matched_deletes = GetMatchedDeletes(*data_entry, delete_file_index);
+ const auto& data_file = data_entry->data_file;
+ tasks.emplace_back(std::make_shared<FileScanTask>(
+ data_file, std::move(matched_deletes), std::move(residual)));
Review Comment:
Yes, this has not been implemented yet, and it is expected that this residual will be shared among all tasks.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "wgtmac (via GitHub)" <gi...@apache.org>.
wgtmac commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2177045928
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+/// \brief Represents a task to scan a table or a portion of it.
Review Comment:
```suggestion
/// \brief An abstract scan task.
```
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+/// \brief Represents a task to scan a table or a portion of it.
+class ICEBERG_EXPORT ScanTask {
+ public:
+ virtual ~ScanTask() = default;
+
+ /// \brief The number of bytes that should be read by this scan task.
+ virtual int64_t size_bytes() const = 0;
+
+ /// \brief The number of files that should be read by this scan task.
+ virtual int32_t files_count() const = 0;
+
+ /// \brief The number of rows that should be read by this scan task.
+ virtual int64_t estimated_row_count() const = 0;
+};
+
+/// \brief Represents a task to scan a portion of a data file.
+class ICEBERG_EXPORT FileScanTask : public ScanTask {
Review Comment:
Some thoughts about `FileScanTask`:
1. Should we remove `ScanTask` abstraction above? If we remove the abstraction, we can directly use aggregate initialization to create a task. Otherwise we may need to expand the constructor every time a new parameter is required.
2. If we do (1) above, is it possible also to make it a simple struct by removing all functions (as they are all trivial accessors).
3. Should we add fields (a.k.a. spec and partition_value) from Java `PartitionScanTask` to support partitioning? We can add them later but a TODO comment is desirable.
4. Should we combine `start` and `length`, and wrap them by `std::optional`? I believe they are not required at all times.
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+/// \brief Represents a task to scan a table or a portion of it.
+class ICEBERG_EXPORT ScanTask {
+ public:
+ virtual ~ScanTask() = default;
+
+ /// \brief The number of bytes that should be read by this scan task.
+ virtual int64_t size_bytes() const = 0;
+
+ /// \brief The number of files that should be read by this scan task.
+ virtual int32_t files_count() const = 0;
+
+ /// \brief The number of rows that should be read by this scan task.
+ virtual int64_t estimated_row_count() const = 0;
+};
+
+/// \brief Represents a task to scan a portion of a data file.
Review Comment:
```suggestion
/// \brief Task representing a data file and its corresponding delete files.
```
Copied from `iceberg-python`
##########
src/iceberg/manifest_reader.h:
##########
@@ -43,10 +44,27 @@ class ICEBERG_EXPORT ManifestReader {
/// \brief Read manifest files from a manifest list file.
class ICEBERG_EXPORT ManifestListReader {
public:
+ virtual ~ManifestListReader() = default;
virtual Result<std::span<std::unique_ptr<ManifestFile>>> Files() const = 0;
private:
std::unique_ptr<StructLikeReader> reader_;
};
+/// \brief Creates a reader for the manifest list.
+/// \param file_path Path to the manifest list file.
+/// \return A Result containing the reader or an error.
+Result<std::unique_ptr<ManifestListReader>> CreateManifestListReader(
+ const std::string& file_path) {
Review Comment:
I don't know yet. It depends on how we will use them. BTW @dongxiao1198 will work on manifest reading.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "wgtmac (via GitHub)" <gi...@apache.org>.
wgtmac commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2181398464
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+/// \brief An abstract scan task.
+class ICEBERG_EXPORT ScanTask {
+ public:
+ virtual ~ScanTask() = default;
+
+ /// \brief The number of bytes that should be read by this scan task.
+ virtual int64_t SizeBytes() const = 0;
+
+ /// \brief The number of files that should be read by this scan task.
+ virtual int32_t FilesCount() const = 0;
+
+ /// \brief The number of rows that should be read by this scan task.
+ virtual int64_t EstimatedRowCount() const = 0;
+};
+
+/// \brief Task representing a data file and its corresponding delete files.
+class ICEBERG_EXPORT FileScanTask : public ScanTask {
+ public:
+ FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files, int64_t start,
+ int64_t length, std::shared_ptr<Expression> residual);
+
+ /// \brief The data file that should be read by this scan task.
+ const std::shared_ptr<DataFile>& data_file() const;
+
+ /// \brief The delete files that should be read by this scan task.
+ const std::vector<std::shared_ptr<DataFile>>& delete_files() const;
+
+ /// \brief The byte offset in the data file where the scan should start.
+ int64_t start() const;
+
+ /// \brief The length in bytes to scan from the start offset.
+ int64_t length() const;
+
+ /// \brief The residual expression to apply after scanning the data file.
+ const std::shared_ptr<Expression>& residual() const;
+
+ int64_t SizeBytes() const override;
+ int32_t FilesCount() const override;
+ int64_t EstimatedRowCount() const override;
+
+ private:
+ /// \brief Data file metadata.
+ std::shared_ptr<DataFile> data_file_;
+ /// \brief Delete files metadata.
+ std::vector<std::shared_ptr<DataFile>> delete_files_;
+ /// \brief Start byte offset.
+ int64_t start_;
+ /// \brief Length in bytes to scan.
+ int64_t length_;
+ /// \brief Residual expression to apply.
+ std::shared_ptr<Expression> residual_;
+};
+
+/// \brief Scan context holding snapshot and scan-specific metadata.
+struct TableScanContext {
+ /// \brief Table metadata.
+ std::shared_ptr<TableMetadata> table_metadata;
+ /// \brief Snapshot to scan.
+ std::shared_ptr<Snapshot> snapshot;
+ /// \brief Projected schema.
+ std::shared_ptr<Schema> projected_schema;
+ /// \brief Filter expression to apply.
+ std::shared_ptr<Expression> filter;
+ /// \brief Whether the scan is case-sensitive.
+ bool case_sensitive = false;
+ /// \brief Additional options for the scan.
+ std::unordered_map<std::string, std::string> options;
+ /// \brief Optional limit on the number of rows to scan.
+ std::optional<int64_t> limit;
+};
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table_metadata The metadata of the table to scan.
+ /// \param file_io The FileIO instance for reading manifests and data files.
+ explicit TableScanBuilder(std::shared_ptr<TableMetadata> table_metadata,
+ std::shared_ptr<FileIO> file_io);
+
+ /// \brief Sets the snapshot ID to scan.
+ /// \param snapshot_id The ID of the snapshot.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSnapshotId(int64_t snapshot_id);
+
+ /// \brief Selects columns to include in the scan.
+ /// \param column_names A list of column names. If empty, all columns will be selected.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithColumnNames(std::vector<std::string> column_names);
+
+ /// \brief Sets the schema to use for the scan.
+ /// \param schema The schema to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithProjectedSchema(std::shared_ptr<Schema> schema);
+
+ /// \brief Applies a filter expression to the scan.
+ /// \param filter Filter expression to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithFilter(std::shared_ptr<Expression> filter);
+
+ /// \brief Sets whether the scan should be case-sensitive.
+ /// \param case_sensitive Whether the scan is case-sensitive.
+ /// /return Reference to the builder.
+ TableScanBuilder& WithCaseSensitive(bool case_sensitive);
+
+ /// \brief Sets an option for the scan.
+ /// \param property The name of the option.
+ /// \param value The value of the option.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithOption(std::string property, std::string value);
+
+ /// \brief Sets an optional limit on the number of rows to scan.
+ /// \param limit Optional limit on the number of rows.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithLimit(std::optional<int64_t> limit);
+
+ /// \brief Builds and returns a TableScan instance.
+ /// \return A Result containing the TableScan or an error.
+ Result<std::unique_ptr<TableScan>> Build();
+
+ private:
+ /// \brief the file I/O instance for reading manifests and data files.
+ std::shared_ptr<FileIO> file_io_;
+ /// \brief column names to project in the scan.
+ std::vector<std::string> column_names_;
+ /// \brief snapshot ID to scan, if specified.
+ std::optional<int64_t> snapshot_id_;
+ /// \brief Context for the scan, including snapshot, schema, and filter.
+ TableScanContext context_;
+};
+
+/// \brief Represents a configured scan operation on a table.
+class ICEBERG_EXPORT TableScan {
+ public:
+ virtual ~TableScan() = default;
+
+ /// \brief Constructs a TableScan with the given context and file I/O.
+ /// \param context Scan context including snapshot, schema, and filter.
+ /// \param file_io File I/O instance for reading manifests and data files.
+ TableScan(TableScanContext context, std::shared_ptr<FileIO> file_io);
+
+ /// \brief Returns the snapshot being scanned.
+ /// \return A shared pointer to the snapshot.
+ const std::shared_ptr<Snapshot>& snapshot() const;
+
+ /// \brief Returns the projected schema for the scan.
+ /// \return A shared pointer to the projected schema.
+ const std::shared_ptr<Schema>& projection() const;
+
+ /// \brief Returns the scan context.
+ /// \return A reference to the TableScanContext.
+ const TableScanContext& context() const;
+
+ /// \brief Returns the file I/O instance used for reading manifests and data files.
+ /// \return A shared pointer to the FileIO instance.
+ const std::shared_ptr<FileIO>& io() const;
+
+ /// \brief Plans the scan tasks by resolving manifests and data files.
+ /// \return A Result containing scan tasks or an error.
+ virtual Result<std::vector<std::shared_ptr<FileScanTask>>> PlanFiles() const = 0;
+
+ protected:
+ /// \brief context for the scan, including snapshot, schema, and filter.
+ const TableScanContext context_;
+ /// \brief File I/O instance for reading manifests and data files.
+ std::shared_ptr<FileIO> file_io_;
+};
+
+/// \brief A scan that reads data files and applies delete files to filter rows.
+class ICEBERG_EXPORT DataScan : public TableScan {
Review Comment:
I'm a little bit confused about the name of Scan and ScanTask across different implementations. Should this be `DataTableScan` which produces `FileScanTask`? For `DataScan`, I think it should produce a group of `DataTask` which contains rows of `FileScanTask`.
Simply put:
Scan -> ScanTask
TableScan -> FileScanTask
DataScan -> DataTask
DataScan inherits TableScan inherits Scan
DataTask inherits FileScanTask inherits ScanTask
WDYT? @gty404 @Fokko
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "gty404 (via GitHub)" <gi...@apache.org>.
gty404 commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2188793310
##########
src/iceberg/table.cc:
##########
@@ -107,4 +108,8 @@ const std::vector<SnapshotLogEntry>& Table::history() const {
const std::shared_ptr<FileIO>& Table::io() const { return io_; }
+std::unique_ptr<TableScanBuilder> Table::NewScan() const {
+ return std::make_unique<TableScanBuilder>(metadata_, io_);
Review Comment:
If I pass "Table" to the "TableScanBuilder," I cannot pass it further to "DataTableScan," as "Table" can only be passed by reference to the "TableScanBuilder."
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "zhjwpku (via GitHub)" <gi...@apache.org>.
zhjwpku commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2191673278
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,285 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+namespace {
+/// \brief Use indexed data structures for efficient lookups
+class DeleteFileIndex {
+ public:
+ /// \brief Build the index from a list of manifest entries.
+ explicit DeleteFileIndex(const std::vector<std::unique_ptr<ManifestEntry>>& entries) {
+ for (const auto& entry : entries) {
+ const int64_t seq_num =
+ entry->sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ sequence_index.emplace(seq_num, entry.get());
+ }
+ }
+
+ /// \brief Find delete files that match the sequence number of a data entry.
+ std::vector<ManifestEntry*> FindRelevantEntries(const ManifestEntry& data_entry) const {
+ std::vector<ManifestEntry*> relevant_deletes;
+
+ // Use lower_bound for efficient range search
+ auto data_sequence_number =
+ data_entry.sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ for (auto it = sequence_index.lower_bound(data_sequence_number);
+ it != sequence_index.end(); ++it) {
+ // Additional filtering logic here
+ relevant_deletes.push_back(it->second);
+ }
+
+ return relevant_deletes;
+ }
+
+ private:
+ /// \brief Index by sequence number for quick filtering
+ std::multimap<int64_t, ManifestEntry*> sequence_index;
+};
+
+/// \brief Get matched delete files for a given data entry.
+std::vector<std::shared_ptr<DataFile>> GetMatchedDeletes(
+ const ManifestEntry& data_entry, const DeleteFileIndex& delete_file_index) {
+ const auto relevant_entries = delete_file_index.FindRelevantEntries(data_entry);
+ std::vector<std::shared_ptr<DataFile>> matched_deletes;
+ if (relevant_entries.empty()) {
+ return matched_deletes;
+ }
+
+ matched_deletes.reserve(relevant_entries.size());
+ for (const auto& delete_entry : relevant_entries) {
+ // TODO(gty404): check if the delete entry contains the data entry's file path
+ matched_deletes.emplace_back(delete_entry->data_file);
+ }
+ return matched_deletes;
+}
+} // namespace
+
+// implement FileScanTask
+FileScanTask::FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files,
+ std::shared_ptr<Expression> residual)
+ : data_file_(std::move(file)),
+ delete_files_(std::move(delete_files)),
+ residual_(std::move(residual)) {}
+
+const std::shared_ptr<DataFile>& FileScanTask::data_file() const { return data_file_; }
+
+const std::vector<std::shared_ptr<DataFile>>& FileScanTask::delete_files() const {
+ return delete_files_;
+}
+
+int64_t FileScanTask::SizeBytes() const {
+ int64_t size_in_bytes = data_file_->file_size_in_bytes;
+ std::ranges::for_each(delete_files_, [&size_in_bytes](const auto& delete_file) {
+ size_in_bytes += delete_file->file_size_in_bytes;
+ });
+ return size_in_bytes;
+}
+
+int32_t FileScanTask::FilesCount() const {
+ return static_cast<int32_t>(delete_files_.size() + 1);
+}
+
+int64_t FileScanTask::EstimatedRowCount() const {
+ if (data_file_->file_size_in_bytes == 0) {
+ return 0;
+ }
+ const auto size_in_bytes = data_file_->file_size_in_bytes;
+ const double scannedFileFraction =
+ static_cast<double>(size_in_bytes) / data_file_->file_size_in_bytes;
+ return static_cast<int64_t>(scannedFileFraction * data_file_->record_count);
Review Comment:
I don't understand the math here, shouldn't the result be `data_file_->record_count - sum(delete_file.record_count)`?
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,285 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+namespace {
+/// \brief Use indexed data structures for efficient lookups
+class DeleteFileIndex {
+ public:
+ /// \brief Build the index from a list of manifest entries.
+ explicit DeleteFileIndex(const std::vector<std::unique_ptr<ManifestEntry>>& entries) {
+ for (const auto& entry : entries) {
+ const int64_t seq_num =
+ entry->sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ sequence_index.emplace(seq_num, entry.get());
+ }
+ }
+
+ /// \brief Find delete files that match the sequence number of a data entry.
+ std::vector<ManifestEntry*> FindRelevantEntries(const ManifestEntry& data_entry) const {
+ std::vector<ManifestEntry*> relevant_deletes;
+
+ // Use lower_bound for efficient range search
+ auto data_sequence_number =
+ data_entry.sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ for (auto it = sequence_index.lower_bound(data_sequence_number);
+ it != sequence_index.end(); ++it) {
+ // Additional filtering logic here
+ relevant_deletes.push_back(it->second);
+ }
+
+ return relevant_deletes;
+ }
+
+ private:
+ /// \brief Index by sequence number for quick filtering
+ std::multimap<int64_t, ManifestEntry*> sequence_index;
+};
+
+/// \brief Get matched delete files for a given data entry.
+std::vector<std::shared_ptr<DataFile>> GetMatchedDeletes(
+ const ManifestEntry& data_entry, const DeleteFileIndex& delete_file_index) {
+ const auto relevant_entries = delete_file_index.FindRelevantEntries(data_entry);
+ std::vector<std::shared_ptr<DataFile>> matched_deletes;
+ if (relevant_entries.empty()) {
+ return matched_deletes;
+ }
+
+ matched_deletes.reserve(relevant_entries.size());
+ for (const auto& delete_entry : relevant_entries) {
+ // TODO(gty404): check if the delete entry contains the data entry's file path
+ matched_deletes.emplace_back(delete_entry->data_file);
+ }
+ return matched_deletes;
+}
+} // namespace
+
+// implement FileScanTask
+FileScanTask::FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files,
+ std::shared_ptr<Expression> residual)
+ : data_file_(std::move(file)),
+ delete_files_(std::move(delete_files)),
+ residual_(std::move(residual)) {}
+
+const std::shared_ptr<DataFile>& FileScanTask::data_file() const { return data_file_; }
+
+const std::vector<std::shared_ptr<DataFile>>& FileScanTask::delete_files() const {
+ return delete_files_;
+}
+
+int64_t FileScanTask::SizeBytes() const {
+ int64_t size_in_bytes = data_file_->file_size_in_bytes;
+ std::ranges::for_each(delete_files_, [&size_in_bytes](const auto& delete_file) {
+ size_in_bytes += delete_file->file_size_in_bytes;
+ });
+ return size_in_bytes;
+}
+
+int32_t FileScanTask::FilesCount() const {
+ return static_cast<int32_t>(delete_files_.size() + 1);
+}
+
+int64_t FileScanTask::EstimatedRowCount() const {
+ if (data_file_->file_size_in_bytes == 0) {
+ return 0;
+ }
+ const auto size_in_bytes = data_file_->file_size_in_bytes;
+ const double scannedFileFraction =
+ static_cast<double>(size_in_bytes) / data_file_->file_size_in_bytes;
+ return static_cast<int64_t>(scannedFileFraction * data_file_->record_count);
+}
+
+const std::shared_ptr<Expression>& FileScanTask::residual() const { return residual_; }
+
+TableScanBuilder::TableScanBuilder(std::shared_ptr<TableMetadata> table_metadata,
+ std::shared_ptr<FileIO> file_io)
+ : file_io_(std::move(file_io)) {
+ context_.table_metadata = std::move(table_metadata);
+}
+
+TableScanBuilder& TableScanBuilder::WithColumnNames(
+ std::vector<std::string> column_names) {
+ column_names_ = std::move(column_names);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithProjectedSchema(std::shared_ptr<Schema> schema) {
+ context_.projected_schema = std::move(schema);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithSnapshotId(int64_t snapshot_id) {
+ snapshot_id_ = snapshot_id;
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithFilter(std::shared_ptr<Expression> filter) {
+ context_.filter = std::move(filter);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithCaseSensitive(bool case_sensitive) {
+ context_.case_sensitive = case_sensitive;
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithOption(std::string property, std::string value) {
+ context_.options[std::move(property)] = std::move(value);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithLimit(std::optional<int64_t> limit) {
+ context_.limit = limit;
+ return *this;
+}
+
+Result<std::unique_ptr<TableScan>> TableScanBuilder::Build() {
+ const auto& table_metadata = context_.table_metadata;
+ auto snapshot_id = snapshot_id_ ? snapshot_id_ : table_metadata->current_snapshot_id;
+ if (!snapshot_id) {
+ return InvalidArgument("No snapshot ID specified for table {}",
+ table_metadata->table_uuid);
+ }
+ auto iter = std::ranges::find_if(
+ table_metadata->snapshots,
+ [id = *snapshot_id](const auto& snapshot) { return snapshot->snapshot_id == id; });
+ if (iter == table_metadata->snapshots.end() || *iter == nullptr) {
Review Comment:
Why can iter be nullptr?
```
/// A list of valid snapshots
std::vector<std::shared_ptr<Snapshot>> snapshots;
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "wgtmac (via GitHub)" <gi...@apache.org>.
wgtmac commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2203423647
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+// implement FileScanTask
+FileScanTask::FileScanTask(std::shared_ptr<DataFile> file)
+ : data_file_(std::move(file)) {}
+
+const std::shared_ptr<DataFile>& FileScanTask::data_file() const { return data_file_; }
+
+int64_t FileScanTask::size_bytes() const { return data_file_->file_size_in_bytes; }
+
+int32_t FileScanTask::files_count() const { return 1; }
+
+int64_t FileScanTask::estimated_row_count() const { return data_file_->record_count; }
+
+TableScanBuilder::TableScanBuilder(std::shared_ptr<TableMetadata> table_metadata,
+ std::shared_ptr<FileIO> file_io)
+ : file_io_(std::move(file_io)) {
+ context_.table_metadata = std::move(table_metadata);
+}
+
+TableScanBuilder& TableScanBuilder::WithColumnNames(
+ std::vector<std::string> column_names) {
+ column_names_ = std::move(column_names);
Review Comment:
nit: make sure `context_.projected_schema` is not set.
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+// implement FileScanTask
+FileScanTask::FileScanTask(std::shared_ptr<DataFile> file)
+ : data_file_(std::move(file)) {}
Review Comment:
```suggestion
FileScanTask::FileScanTask(std::shared_ptr<DataFile> data_file)
: data_file_(std::move(data_file)) {}
```
nit: I think eventually we need to rename it to `data_file` once we will add delete files.
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+// implement FileScanTask
+FileScanTask::FileScanTask(std::shared_ptr<DataFile> file)
+ : data_file_(std::move(file)) {}
+
+const std::shared_ptr<DataFile>& FileScanTask::data_file() const { return data_file_; }
+
+int64_t FileScanTask::size_bytes() const { return data_file_->file_size_in_bytes; }
+
+int32_t FileScanTask::files_count() const { return 1; }
+
+int64_t FileScanTask::estimated_row_count() const { return data_file_->record_count; }
+
+TableScanBuilder::TableScanBuilder(std::shared_ptr<TableMetadata> table_metadata,
+ std::shared_ptr<FileIO> file_io)
+ : file_io_(std::move(file_io)) {
+ context_.table_metadata = std::move(table_metadata);
+}
+
+TableScanBuilder& TableScanBuilder::WithColumnNames(
+ std::vector<std::string> column_names) {
+ column_names_ = std::move(column_names);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithProjectedSchema(std::shared_ptr<Schema> schema) {
+ context_.projected_schema = std::move(schema);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithSnapshotId(int64_t snapshot_id) {
+ snapshot_id_ = snapshot_id;
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithFilter(std::shared_ptr<Expression> filter) {
+ context_.filter = std::move(filter);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithCaseSensitive(bool case_sensitive) {
+ context_.case_sensitive = case_sensitive;
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithOption(std::string property, std::string value) {
+ context_.options[std::move(property)] = std::move(value);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithLimit(std::optional<int64_t> limit) {
+ context_.limit = limit;
+ return *this;
+}
+
+Result<std::unique_ptr<TableScan>> TableScanBuilder::Build() {
+ const auto& table_metadata = context_.table_metadata;
+ auto snapshot_id = snapshot_id_ ? snapshot_id_ : table_metadata->current_snapshot_id;
+ if (!snapshot_id) {
+ return InvalidArgument("No snapshot ID specified for table {}",
+ table_metadata->table_uuid);
+ }
+ auto iter = std::ranges::find_if(
Review Comment:
nit: add `Result<std::shared_ptr<Snapshot>> TableMetadata::Snapshot(int64_t snapshot_id) const` and move the logic below to it.
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+// implement FileScanTask
+FileScanTask::FileScanTask(std::shared_ptr<DataFile> file)
+ : data_file_(std::move(file)) {}
+
+const std::shared_ptr<DataFile>& FileScanTask::data_file() const { return data_file_; }
+
+int64_t FileScanTask::size_bytes() const { return data_file_->file_size_in_bytes; }
+
+int32_t FileScanTask::files_count() const { return 1; }
+
+int64_t FileScanTask::estimated_row_count() const { return data_file_->record_count; }
+
+TableScanBuilder::TableScanBuilder(std::shared_ptr<TableMetadata> table_metadata,
+ std::shared_ptr<FileIO> file_io)
+ : file_io_(std::move(file_io)) {
+ context_.table_metadata = std::move(table_metadata);
+}
+
+TableScanBuilder& TableScanBuilder::WithColumnNames(
+ std::vector<std::string> column_names) {
+ column_names_ = std::move(column_names);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithProjectedSchema(std::shared_ptr<Schema> schema) {
+ context_.projected_schema = std::move(schema);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithSnapshotId(int64_t snapshot_id) {
+ snapshot_id_ = snapshot_id;
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithFilter(std::shared_ptr<Expression> filter) {
+ context_.filter = std::move(filter);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithCaseSensitive(bool case_sensitive) {
+ context_.case_sensitive = case_sensitive;
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithOption(std::string property, std::string value) {
+ context_.options[std::move(property)] = std::move(value);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithLimit(std::optional<int64_t> limit) {
+ context_.limit = limit;
+ return *this;
+}
+
+Result<std::unique_ptr<TableScan>> TableScanBuilder::Build() {
+ const auto& table_metadata = context_.table_metadata;
+ auto snapshot_id = snapshot_id_ ? snapshot_id_ : table_metadata->current_snapshot_id;
+ if (!snapshot_id) {
+ return InvalidArgument("No snapshot ID specified for table {}",
+ table_metadata->table_uuid);
+ }
+ auto iter = std::ranges::find_if(
+ table_metadata->snapshots,
+ [id = *snapshot_id](const auto& snapshot) { return snapshot->snapshot_id == id; });
+ if (iter == table_metadata->snapshots.end()) {
+ return NotFound("Snapshot with ID {} is not found", *snapshot_id);
+ }
+ context_.snapshot = *iter;
+
+ if (!context_.projected_schema) {
+ const auto& snapshot = context_.snapshot;
+ auto schema_id =
+ snapshot->schema_id ? snapshot->schema_id : table_metadata->current_schema_id;
+ if (!schema_id) {
+ return InvalidArgument("No schema ID found in snapshot {} for table {}",
+ snapshot->snapshot_id, table_metadata->table_uuid);
+ }
+
+ const auto& schemas = table_metadata->schemas;
+ const auto it = std::ranges::find_if(schemas, [id = *schema_id](const auto& schema) {
Review Comment:
ditto, we can add `TableMetadata::Schema(int64_t schema_id)` for this.
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+// implement FileScanTask
+FileScanTask::FileScanTask(std::shared_ptr<DataFile> file)
+ : data_file_(std::move(file)) {}
+
+const std::shared_ptr<DataFile>& FileScanTask::data_file() const { return data_file_; }
+
+int64_t FileScanTask::size_bytes() const { return data_file_->file_size_in_bytes; }
+
+int32_t FileScanTask::files_count() const { return 1; }
+
+int64_t FileScanTask::estimated_row_count() const { return data_file_->record_count; }
+
+TableScanBuilder::TableScanBuilder(std::shared_ptr<TableMetadata> table_metadata,
+ std::shared_ptr<FileIO> file_io)
+ : file_io_(std::move(file_io)) {
+ context_.table_metadata = std::move(table_metadata);
+}
+
+TableScanBuilder& TableScanBuilder::WithColumnNames(
+ std::vector<std::string> column_names) {
+ column_names_ = std::move(column_names);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithProjectedSchema(std::shared_ptr<Schema> schema) {
+ context_.projected_schema = std::move(schema);
Review Comment:
nit: make sure `column_names_` is not set.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "lidavidm (via GitHub)" <gi...@apache.org>.
lidavidm commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2110599021
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/expression/expression.h"
+#include "iceberg/file_io.h"
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table Reference to the table to scan.
+ explicit TableScanBuilder(const Table& table);
+
+ /// \brief Sets the snapshot ID to scan.
+ /// \param snapshot_id The ID of the snapshot.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSnapshotId(int64_t snapshot_id);
+
+ /// \brief Selects columns to include in the scan.
+ /// \param column_names A list of column names.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithColumnNames(const std::vector<std::string>& column_names);
+
+ /// \brief Applies a filter expression to the scan.
+ /// \param filter Filter expression to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithFilter(const std::shared_ptr<Expression>& filter);
+
+ /// \brief Builds and returns a TableScan instance.
+ /// \return A Result containing the TableScan or an error.
+ Result<std::unique_ptr<TableScan>> Build();
+
+ private:
+ const Table& table_;
+ std::vector<std::string> column_names_;
+ std::optional<int64_t> snapshot_id_;
+ std::shared_ptr<Expression> filter_;
+};
+
+/// \brief Represents a configured scan operation on a table.
+class ICEBERG_EXPORT TableScan {
+ public:
+ /// \brief Scan context holding snapshot and scan-specific metadata.
+ struct ScanContext {
+ std::shared_ptr<Snapshot> snapshot_; ///< Snapshot to scan.
Review Comment:
Same here: these are public fields
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/expression/expression.h"
+#include "iceberg/file_io.h"
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table Reference to the table to scan.
+ explicit TableScanBuilder(const Table& table);
+
+ /// \brief Sets the snapshot ID to scan.
+ /// \param snapshot_id The ID of the snapshot.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSnapshotId(int64_t snapshot_id);
+
+ /// \brief Selects columns to include in the scan.
+ /// \param column_names A list of column names.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithColumnNames(const std::vector<std::string>& column_names);
+
+ /// \brief Applies a filter expression to the scan.
+ /// \param filter Filter expression to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithFilter(const std::shared_ptr<Expression>& filter);
+
+ /// \brief Builds and returns a TableScan instance.
+ /// \return A Result containing the TableScan or an error.
+ Result<std::unique_ptr<TableScan>> Build();
+
+ private:
+ const Table& table_;
+ std::vector<std::string> column_names_;
+ std::optional<int64_t> snapshot_id_;
+ std::shared_ptr<Expression> filter_;
+};
+
+/// \brief Represents a configured scan operation on a table.
+class ICEBERG_EXPORT TableScan {
+ public:
+ /// \brief Scan context holding snapshot and scan-specific metadata.
+ struct ScanContext {
+ std::shared_ptr<Snapshot> snapshot_; ///< Snapshot to scan.
+ std::shared_ptr<Schema> schema_; ///< Projected schema.
+ std::vector<int32_t> field_ids_; ///< Field IDs of selected columns.
+ std::shared_ptr<Expression> filter_; ///< Filter expression to apply.
+ };
+
+ /// \brief Constructs a TableScan with the given context and file I/O.
+ /// \param context Scan context including snapshot, schema, and filter.
+ /// \param file_io File I/O instance for reading manifests and data files.
+ TableScan(std::unique_ptr<ScanContext> context, std::shared_ptr<FileIO> file_io);
Review Comment:
Is there any reason to have unique_ptr vs just the struct itself?
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+TableScanBuilder::TableScanBuilder(const Table& table) : table_(table) {}
+
+TableScanBuilder& TableScanBuilder::WithColumnNames(
+ const std::vector<std::string>& column_names) {
+ column_names_ = column_names;
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithSnapshotId(int64_t snapshot_id) {
+ snapshot_id_ = snapshot_id;
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithFilter(
+ const std::shared_ptr<Expression>& filter) {
+ filter_ = filter;
+ return *this;
+}
+
+Result<std::unique_ptr<TableScan>> TableScanBuilder::Build() {
+ ICEBERG_ASSIGN_OR_RAISE(auto snapshot, snapshot_id_ ? table_.snapshot(*snapshot_id_)
+ : Result<std::shared_ptr<Snapshot>>(
+ table_.current_snapshot()));
+
+ auto ResolveSchema = [&]() -> Result<std::shared_ptr<Schema>> {
+ if (snapshot->schema_id) {
+ const auto& schemas = table_.schemas();
+ if (auto it = schemas.find(*snapshot->schema_id); it != schemas.end()) {
+ return it->second;
+ }
+ return InvalidData("Schema {} in snapshot {} is not found", *snapshot->schema_id,
+ snapshot->snapshot_id);
+ }
+ return table_.schema();
+ };
+
+ ICEBERG_ASSIGN_OR_RAISE(auto schema, ResolveSchema());
Review Comment:
Why are we indirecting through a lambda?
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+TableScanBuilder::TableScanBuilder(const Table& table) : table_(table) {}
+
+TableScanBuilder& TableScanBuilder::WithColumnNames(
+ const std::vector<std::string>& column_names) {
+ column_names_ = column_names;
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithSnapshotId(int64_t snapshot_id) {
+ snapshot_id_ = snapshot_id;
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithFilter(
+ const std::shared_ptr<Expression>& filter) {
+ filter_ = filter;
+ return *this;
+}
+
+Result<std::unique_ptr<TableScan>> TableScanBuilder::Build() {
+ ICEBERG_ASSIGN_OR_RAISE(auto snapshot, snapshot_id_ ? table_.snapshot(*snapshot_id_)
+ : Result<std::shared_ptr<Snapshot>>(
Review Comment:
Might as well write out the if/else at this point
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "Xuanwo (via GitHub)" <gi...@apache.org>.
Xuanwo merged PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "wgtmac (via GitHub)" <gi...@apache.org>.
wgtmac commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2177408751
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+class ICEBERG_EXPORT ScanTask {
+ public:
+ virtual ~ScanTask() = default;
+
+ /// \brief The number of bytes that should be read by this scan task.
+ virtual int64_t size_bytes() const = 0;
+
+ /// \brief The number of files that should be read by this scan task.
+ virtual int32_t files_count() const = 0;
+
+ /// \brief The number of rows that should be read by this scan task.
+ virtual int64_t estimated_row_count() const = 0;
+};
+
+/// \brief Represents a task to scan a portion of a data file.
+class ICEBERG_EXPORT FileScanTask : public ScanTask {
+ public:
+ FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files, int64_t start,
+ int64_t length, std::shared_ptr<Expression> residual);
+
+ /// \brief The data file that should be read by this scan task.
+ const std::shared_ptr<DataFile>& data_file() const;
+
+ /// \brief The delete files that should be read by this scan task.
+ const std::vector<std::shared_ptr<DataFile>>& delete_files() const;
+
+ /// \brief The byte offset in the data file where the scan should start.
+ int64_t start() const;
+
+ /// \brief The length in bytes to scan from the start offset.
+ int64_t length() const;
+
+ /// \brief The residual expression to apply after scanning the data file.
+ const std::shared_ptr<Expression>& residual() const;
+
+ int64_t size_bytes() const override;
+ int32_t files_count() const override;
+ int64_t estimated_row_count() const override;
+
+ private:
+ std::shared_ptr<DataFile> data_file_; ///< Data file metadata.
+ std::vector<std::shared_ptr<DataFile>> delete_files_; ///< Delete files metadata.
+
+ int64_t start_; ///< Start byte offset.
+ int64_t length_; ///< Length in bytes to scan.
+
+ std::shared_ptr<Expression> residual_; ///< Residual expression to apply.
+};
+
+/// \brief Scan context holding snapshot and scan-specific metadata.
+struct TableScanContext {
+ std::shared_ptr<TableMetadata> table_metadata; ///< Table metadata.
+ std::shared_ptr<Snapshot> snapshot; ///< Snapshot to scan.
+ std::shared_ptr<Schema> projected_schema; ///< Projected schema.
+ std::shared_ptr<Expression> filter; ///< Filter expression to apply.
+ bool case_sensitive = false; ///< Whether the scan is case-sensitive.
+ std::unordered_map<std::string, std::string>
+ options; ///< Additional options for the scan.
+ std::optional<int64_t> limit; ///< Optional limit on the number of rows to scan.
+};
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table The table to scan.
+ /// \param table_metadata The metadata of the table to scan.
+ explicit TableScanBuilder(const Table& table,
Review Comment:
`Table` is just a wrapper around `TableMetadata`. I think we could either (1) expose `TableMetadata` via `Table` (and thus remove `table_metadata` from ctor), or (2) remove `table` and add `file_io` to ctor. I prefer (2) here.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "gty404 (via GitHub)" <gi...@apache.org>.
gty404 commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2192265434
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,285 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+namespace {
+/// \brief Use indexed data structures for efficient lookups
+class DeleteFileIndex {
+ public:
+ /// \brief Build the index from a list of manifest entries.
+ explicit DeleteFileIndex(const std::vector<std::unique_ptr<ManifestEntry>>& entries) {
Review Comment:
Alright, I have removed the logic related to deleting files from the current PR.
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,285 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+namespace {
+/// \brief Use indexed data structures for efficient lookups
+class DeleteFileIndex {
+ public:
+ /// \brief Build the index from a list of manifest entries.
+ explicit DeleteFileIndex(const std::vector<std::unique_ptr<ManifestEntry>>& entries) {
+ for (const auto& entry : entries) {
+ const int64_t seq_num =
+ entry->sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ sequence_index.emplace(seq_num, entry.get());
+ }
+ }
+
+ /// \brief Find delete files that match the sequence number of a data entry.
+ std::vector<ManifestEntry*> FindRelevantEntries(const ManifestEntry& data_entry) const {
+ std::vector<ManifestEntry*> relevant_deletes;
+
+ // Use lower_bound for efficient range search
+ auto data_sequence_number =
+ data_entry.sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ for (auto it = sequence_index.lower_bound(data_sequence_number);
+ it != sequence_index.end(); ++it) {
+ // Additional filtering logic here
+ relevant_deletes.push_back(it->second);
+ }
+
+ return relevant_deletes;
+ }
+
+ private:
+ /// \brief Index by sequence number for quick filtering
+ std::multimap<int64_t, ManifestEntry*> sequence_index;
+};
+
+/// \brief Get matched delete files for a given data entry.
+std::vector<std::shared_ptr<DataFile>> GetMatchedDeletes(
+ const ManifestEntry& data_entry, const DeleteFileIndex& delete_file_index) {
+ const auto relevant_entries = delete_file_index.FindRelevantEntries(data_entry);
+ std::vector<std::shared_ptr<DataFile>> matched_deletes;
+ if (relevant_entries.empty()) {
+ return matched_deletes;
+ }
+
+ matched_deletes.reserve(relevant_entries.size());
+ for (const auto& delete_entry : relevant_entries) {
+ // TODO(gty404): check if the delete entry contains the data entry's file path
+ matched_deletes.emplace_back(delete_entry->data_file);
+ }
+ return matched_deletes;
+}
+} // namespace
+
+// implement FileScanTask
+FileScanTask::FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files,
+ std::shared_ptr<Expression> residual)
+ : data_file_(std::move(file)),
+ delete_files_(std::move(delete_files)),
+ residual_(std::move(residual)) {}
+
+const std::shared_ptr<DataFile>& FileScanTask::data_file() const { return data_file_; }
+
+const std::vector<std::shared_ptr<DataFile>>& FileScanTask::delete_files() const {
+ return delete_files_;
+}
+
+int64_t FileScanTask::SizeBytes() const {
+ int64_t size_in_bytes = data_file_->file_size_in_bytes;
+ std::ranges::for_each(delete_files_, [&size_in_bytes](const auto& delete_file) {
+ size_in_bytes += delete_file->file_size_in_bytes;
+ });
+ return size_in_bytes;
+}
+
+int32_t FileScanTask::FilesCount() const {
+ return static_cast<int32_t>(delete_files_.size() + 1);
+}
+
+int64_t FileScanTask::EstimatedRowCount() const {
+ if (data_file_->file_size_in_bytes == 0) {
+ return 0;
+ }
+ const auto size_in_bytes = data_file_->file_size_in_bytes;
+ const double scannedFileFraction =
+ static_cast<double>(size_in_bytes) / data_file_->file_size_in_bytes;
+ return static_cast<int64_t>(scannedFileFraction * data_file_->record_count);
Review Comment:
Previously, FileScanTask might correspond to a part of DataFile's data, so we used the ratio of the data read to estimate the number of records. Now, there is indeed a problem with this approach, so I have made the necessary modifications.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "lishuxu (via GitHub)" <gi...@apache.org>.
lishuxu commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2173350649
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+class ICEBERG_EXPORT ScanTask {
+ public:
+ virtual ~ScanTask() = default;
+
+ /// \brief The number of bytes that should be read by this scan task.
+ virtual int64_t size_bytes() const = 0;
+
+ /// \brief The number of files that should be read by this scan task.
+ virtual int32_t files_count() const = 0;
+
+ /// \brief The number of rows that should be read by this scan task.
+ virtual int64_t estimated_row_count() const = 0;
+};
+
+/// \brief Represents a task to scan a portion of a data file.
+class ICEBERG_EXPORT FileScanTask : public ScanTask {
+ public:
+ FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files, int64_t start,
+ int64_t length, std::shared_ptr<Expression> residual);
+
+ /// \brief The data file that should be read by this scan task.
+ const std::shared_ptr<DataFile>& data_file() const;
+
+ /// \brief The delete files that should be read by this scan task.
+ const std::vector<std::shared_ptr<DataFile>>& delete_files() const;
+
+ /// \brief The byte offset in the data file where the scan should start.
+ int64_t start() const;
+
+ /// \brief The length in bytes to scan from the start offset.
+ int64_t length() const;
+
+ /// \brief The residual expression to apply after scanning the data file.
+ const std::shared_ptr<Expression>& residual() const;
+
+ int64_t size_bytes() const override;
+ int32_t files_count() const override;
+ int64_t estimated_row_count() const override;
+
+ private:
+ std::shared_ptr<DataFile> data_file_; ///< Data file metadata.
+ std::vector<std::shared_ptr<DataFile>> delete_files_; ///< Delete files metadata.
+
+ int64_t start_; ///< Start byte offset.
+ int64_t length_; ///< Length in bytes to scan.
+
+ std::shared_ptr<Expression> residual_; ///< Residual expression to apply.
+};
+
+/// \brief Scan context holding snapshot and scan-specific metadata.
+struct TableScanContext {
+ std::shared_ptr<TableMetadata> table_metadata; ///< Table metadata.
+ std::shared_ptr<Snapshot> snapshot; ///< Snapshot to scan.
+ std::shared_ptr<Schema> projected_schema; ///< Projected schema.
+ std::shared_ptr<Expression> filter; ///< Filter expression to apply.
+ bool case_sensitive = false; ///< Whether the scan is case-sensitive.
+ std::unordered_map<std::string, std::string>
+ options; ///< Additional options for the scan.
+ std::optional<int64_t> limit; ///< Optional limit on the number of rows to scan.
+};
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table The table to scan.
+ /// \param table_metadata The metadata of the table to scan.
+ explicit TableScanBuilder(const Table& table,
+ std::shared_ptr<TableMetadata> table_metadata);
+
+ /// \brief Sets the snapshot ID to scan.
+ /// \param snapshot_id The ID of the snapshot.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSnapshotId(int64_t snapshot_id);
+
+ /// \brief Selects columns to include in the scan.
+ /// \param column_names A list of column names. If empty, all columns will be selected.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithColumnNames(std::vector<std::string> column_names);
+
+ /// \brief Sets the schema to use for the scan.
+ /// \param schema The schema to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSchema(std::shared_ptr<Schema> schema);
+
+ /// \brief Applies a filter expression to the scan.
+ /// \param filter Filter expression to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithFilter(std::shared_ptr<Expression> filter);
+
+ /// \brief Sets whether the scan should be case-sensitive.
+ /// \param case_sensitive Whether the scan is case-sensitive.
+ /// /return Reference to the builder.
+ TableScanBuilder& WithCaseSensitive(bool case_sensitive);
+
+ /// \brief Sets an option for the scan.
+ /// \param property The name of the option.
+ /// \param value The value of the option.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithOption(std::string property, std::string value);
+
+ /// \brief Sets an optional limit on the number of rows to scan.
+ /// \param limit Optional limit on the number of rows.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithLimit(std::optional<int64_t> limit);
+
+ /// \brief Builds and returns a TableScan instance.
+ /// \return A Result containing the TableScan or an error.
+ Result<std::unique_ptr<TableScan>> Build();
+
+ private:
+ const Table& table_; ///< Reference to the table to scan.
+ std::vector<std::string> column_names_;
+ std::optional<int64_t> snapshot_id_;
+ TableScanContext context_; ///< Context for the scan.
+};
+
+/// \brief Represents a configured scan operation on a table.
+class ICEBERG_EXPORT TableScan {
+ public:
+ virtual ~TableScan() = default;
+
+ /// \brief Constructs a TableScan with the given context and file I/O.
+ /// \param context Scan context including snapshot, schema, and filter.
+ /// \param file_io File I/O instance for reading manifests and data files.
+ TableScan(TableScanContext context, std::shared_ptr<FileIO> file_io);
+
+ /// \brief Returns the snapshot being scanned.
+ /// \return A shared pointer to the snapshot.
+ const std::shared_ptr<Snapshot>& snapshot() const;
+
+ /// \brief Returns the projected schema for the scan.
+ /// \return A shared pointer to the projected schema.
+ const std::shared_ptr<Schema>& projection() const;
+
+ /// \brief Returns the scan context.
+ /// \return A reference to the TableScanContext.
+ const TableScanContext& context() const;
+
+ /// \brief Returns the file I/O instance used for reading manifests and data files.
+ /// \return A shared pointer to the FileIO instance.
+ const std::shared_ptr<FileIO>& io() const;
+
+ /// \brief Plans the scan tasks by resolving manifests and data files.
+ /// \return A Result containing scan tasks or an error.
+ virtual Result<std::vector<std::shared_ptr<FileScanTask>>> PlanFiles() const = 0;
Review Comment:
Would it be more appropriate to name this method as PlanTasks?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "gty404 (via GitHub)" <gi...@apache.org>.
gty404 commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2181763049
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+/// \brief An abstract scan task.
+class ICEBERG_EXPORT ScanTask {
+ public:
+ virtual ~ScanTask() = default;
+
+ /// \brief The number of bytes that should be read by this scan task.
+ virtual int64_t SizeBytes() const = 0;
+
+ /// \brief The number of files that should be read by this scan task.
+ virtual int32_t FilesCount() const = 0;
+
+ /// \brief The number of rows that should be read by this scan task.
+ virtual int64_t EstimatedRowCount() const = 0;
+};
+
+/// \brief Task representing a data file and its corresponding delete files.
+class ICEBERG_EXPORT FileScanTask : public ScanTask {
+ public:
+ FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files, int64_t start,
+ int64_t length, std::shared_ptr<Expression> residual);
+
+ /// \brief The data file that should be read by this scan task.
+ const std::shared_ptr<DataFile>& data_file() const;
+
+ /// \brief The delete files that should be read by this scan task.
+ const std::vector<std::shared_ptr<DataFile>>& delete_files() const;
+
+ /// \brief The byte offset in the data file where the scan should start.
+ int64_t start() const;
+
+ /// \brief The length in bytes to scan from the start offset.
+ int64_t length() const;
+
+ /// \brief The residual expression to apply after scanning the data file.
+ const std::shared_ptr<Expression>& residual() const;
+
+ int64_t SizeBytes() const override;
+ int32_t FilesCount() const override;
+ int64_t EstimatedRowCount() const override;
+
+ private:
+ /// \brief Data file metadata.
+ std::shared_ptr<DataFile> data_file_;
+ /// \brief Delete files metadata.
+ std::vector<std::shared_ptr<DataFile>> delete_files_;
+ /// \brief Start byte offset.
+ int64_t start_;
+ /// \brief Length in bytes to scan.
+ int64_t length_;
+ /// \brief Residual expression to apply.
+ std::shared_ptr<Expression> residual_;
+};
+
+/// \brief Scan context holding snapshot and scan-specific metadata.
+struct TableScanContext {
+ /// \brief Table metadata.
+ std::shared_ptr<TableMetadata> table_metadata;
+ /// \brief Snapshot to scan.
+ std::shared_ptr<Snapshot> snapshot;
+ /// \brief Projected schema.
+ std::shared_ptr<Schema> projected_schema;
+ /// \brief Filter expression to apply.
+ std::shared_ptr<Expression> filter;
+ /// \brief Whether the scan is case-sensitive.
+ bool case_sensitive = false;
+ /// \brief Additional options for the scan.
+ std::unordered_map<std::string, std::string> options;
+ /// \brief Optional limit on the number of rows to scan.
+ std::optional<int64_t> limit;
+};
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table_metadata The metadata of the table to scan.
+ /// \param file_io The FileIO instance for reading manifests and data files.
+ explicit TableScanBuilder(std::shared_ptr<TableMetadata> table_metadata,
+ std::shared_ptr<FileIO> file_io);
+
+ /// \brief Sets the snapshot ID to scan.
+ /// \param snapshot_id The ID of the snapshot.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSnapshotId(int64_t snapshot_id);
+
+ /// \brief Selects columns to include in the scan.
+ /// \param column_names A list of column names. If empty, all columns will be selected.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithColumnNames(std::vector<std::string> column_names);
+
+ /// \brief Sets the schema to use for the scan.
+ /// \param schema The schema to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithProjectedSchema(std::shared_ptr<Schema> schema);
+
+ /// \brief Applies a filter expression to the scan.
+ /// \param filter Filter expression to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithFilter(std::shared_ptr<Expression> filter);
+
+ /// \brief Sets whether the scan should be case-sensitive.
+ /// \param case_sensitive Whether the scan is case-sensitive.
+ /// /return Reference to the builder.
+ TableScanBuilder& WithCaseSensitive(bool case_sensitive);
+
+ /// \brief Sets an option for the scan.
+ /// \param property The name of the option.
+ /// \param value The value of the option.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithOption(std::string property, std::string value);
+
+ /// \brief Sets an optional limit on the number of rows to scan.
+ /// \param limit Optional limit on the number of rows.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithLimit(std::optional<int64_t> limit);
+
+ /// \brief Builds and returns a TableScan instance.
+ /// \return A Result containing the TableScan or an error.
+ Result<std::unique_ptr<TableScan>> Build();
+
+ private:
+ /// \brief the file I/O instance for reading manifests and data files.
+ std::shared_ptr<FileIO> file_io_;
+ /// \brief column names to project in the scan.
+ std::vector<std::string> column_names_;
+ /// \brief snapshot ID to scan, if specified.
+ std::optional<int64_t> snapshot_id_;
+ /// \brief Context for the scan, including snapshot, schema, and filter.
+ TableScanContext context_;
+};
+
+/// \brief Represents a configured scan operation on a table.
+class ICEBERG_EXPORT TableScan {
+ public:
+ virtual ~TableScan() = default;
+
+ /// \brief Constructs a TableScan with the given context and file I/O.
+ /// \param context Scan context including snapshot, schema, and filter.
+ /// \param file_io File I/O instance for reading manifests and data files.
+ TableScan(TableScanContext context, std::shared_ptr<FileIO> file_io);
+
+ /// \brief Returns the snapshot being scanned.
+ /// \return A shared pointer to the snapshot.
+ const std::shared_ptr<Snapshot>& snapshot() const;
+
+ /// \brief Returns the projected schema for the scan.
+ /// \return A shared pointer to the projected schema.
+ const std::shared_ptr<Schema>& projection() const;
+
+ /// \brief Returns the scan context.
+ /// \return A reference to the TableScanContext.
+ const TableScanContext& context() const;
+
+ /// \brief Returns the file I/O instance used for reading manifests and data files.
+ /// \return A shared pointer to the FileIO instance.
+ const std::shared_ptr<FileIO>& io() const;
+
+ /// \brief Plans the scan tasks by resolving manifests and data files.
+ /// \return A Result containing scan tasks or an error.
+ virtual Result<std::vector<std::shared_ptr<FileScanTask>>> PlanFiles() const = 0;
Review Comment:
In iceberg-java, the result of planFiles is a one-to-one correspondence between the entire file and FileScanTask. planTask is the result of splitting the result of planFiles. Each scanTask may contain multiple files, or only a part of one file. I am not sure if the split feature is needed now, so I did not use PlanTask.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "gty404 (via GitHub)" <gi...@apache.org>.
gty404 commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2174080643
##########
src/iceberg/manifest_reader.h:
##########
@@ -43,10 +44,27 @@ class ICEBERG_EXPORT ManifestReader {
/// \brief Read manifest files from a manifest list file.
class ICEBERG_EXPORT ManifestListReader {
public:
+ virtual ~ManifestListReader() = default;
virtual Result<std::span<std::unique_ptr<ManifestFile>>> Files() const = 0;
private:
std::unique_ptr<StructLikeReader> reader_;
};
+/// \brief Creates a reader for the manifest list.
+/// \param file_path Path to the manifest list file.
+/// \return A Result containing the reader or an error.
+Result<std::unique_ptr<ManifestListReader>> CreateManifestListReader(
+ const std::string& file_path) {
Review Comment:
Do you need to provide a ManifestListReaderBuilder/ManifestReaderBuilder?
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+class ICEBERG_EXPORT ScanTask {
+ public:
+ virtual ~ScanTask() = default;
+
+ /// \brief The number of bytes that should be read by this scan task.
+ virtual int64_t size_bytes() const = 0;
+
+ /// \brief The number of files that should be read by this scan task.
+ virtual int32_t files_count() const = 0;
+
+ /// \brief The number of rows that should be read by this scan task.
+ virtual int64_t estimated_row_count() const = 0;
+};
+
+/// \brief Represents a task to scan a portion of a data file.
+class ICEBERG_EXPORT FileScanTask : public ScanTask {
+ public:
+ FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files, int64_t start,
+ int64_t length, std::shared_ptr<Expression> residual);
+
+ /// \brief The data file that should be read by this scan task.
+ const std::shared_ptr<DataFile>& data_file() const;
+
+ /// \brief The delete files that should be read by this scan task.
+ const std::vector<std::shared_ptr<DataFile>>& delete_files() const;
+
+ /// \brief The byte offset in the data file where the scan should start.
+ int64_t start() const;
+
+ /// \brief The length in bytes to scan from the start offset.
+ int64_t length() const;
+
+ /// \brief The residual expression to apply after scanning the data file.
+ const std::shared_ptr<Expression>& residual() const;
+
+ int64_t size_bytes() const override;
+ int32_t files_count() const override;
+ int64_t estimated_row_count() const override;
+
+ private:
+ std::shared_ptr<DataFile> data_file_; ///< Data file metadata.
+ std::vector<std::shared_ptr<DataFile>> delete_files_; ///< Delete files metadata.
Review Comment:
We have not explicitly defined DeleteFile, and from the definition of DataFile, it seems to include DeleteFile?
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+class ICEBERG_EXPORT ScanTask {
+ public:
+ virtual ~ScanTask() = default;
+
+ /// \brief The number of bytes that should be read by this scan task.
+ virtual int64_t size_bytes() const = 0;
+
+ /// \brief The number of files that should be read by this scan task.
+ virtual int32_t files_count() const = 0;
+
+ /// \brief The number of rows that should be read by this scan task.
+ virtual int64_t estimated_row_count() const = 0;
+};
+
+/// \brief Represents a task to scan a portion of a data file.
+class ICEBERG_EXPORT FileScanTask : public ScanTask {
+ public:
+ FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files, int64_t start,
+ int64_t length, std::shared_ptr<Expression> residual);
+
+ /// \brief The data file that should be read by this scan task.
+ const std::shared_ptr<DataFile>& data_file() const;
+
+ /// \brief The delete files that should be read by this scan task.
+ const std::vector<std::shared_ptr<DataFile>>& delete_files() const;
+
+ /// \brief The byte offset in the data file where the scan should start.
+ int64_t start() const;
+
+ /// \brief The length in bytes to scan from the start offset.
+ int64_t length() const;
+
+ /// \brief The residual expression to apply after scanning the data file.
+ const std::shared_ptr<Expression>& residual() const;
+
+ int64_t size_bytes() const override;
+ int32_t files_count() const override;
+ int64_t estimated_row_count() const override;
+
+ private:
+ std::shared_ptr<DataFile> data_file_; ///< Data file metadata.
+ std::vector<std::shared_ptr<DataFile>> delete_files_; ///< Delete files metadata.
+
+ int64_t start_; ///< Start byte offset.
+ int64_t length_; ///< Length in bytes to scan.
+
+ std::shared_ptr<Expression> residual_; ///< Residual expression to apply.
+};
+
+/// \brief Scan context holding snapshot and scan-specific metadata.
+struct TableScanContext {
+ std::shared_ptr<TableMetadata> table_metadata; ///< Table metadata.
+ std::shared_ptr<Snapshot> snapshot; ///< Snapshot to scan.
+ std::shared_ptr<Schema> projected_schema; ///< Projected schema.
+ std::shared_ptr<Expression> filter; ///< Filter expression to apply.
+ bool case_sensitive = false; ///< Whether the scan is case-sensitive.
+ std::unordered_map<std::string, std::string>
+ options; ///< Additional options for the scan.
+ std::optional<int64_t> limit; ///< Optional limit on the number of rows to scan.
+};
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table The table to scan.
+ /// \param table_metadata The metadata of the table to scan.
+ explicit TableScanBuilder(const Table& table,
+ std::shared_ptr<TableMetadata> table_metadata);
+
+ /// \brief Sets the snapshot ID to scan.
+ /// \param snapshot_id The ID of the snapshot.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSnapshotId(int64_t snapshot_id);
+
+ /// \brief Selects columns to include in the scan.
+ /// \param column_names A list of column names. If empty, all columns will be selected.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithColumnNames(std::vector<std::string> column_names);
+
+ /// \brief Sets the schema to use for the scan.
+ /// \param schema The schema to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSchema(std::shared_ptr<Schema> schema);
+
+ /// \brief Applies a filter expression to the scan.
+ /// \param filter Filter expression to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithFilter(std::shared_ptr<Expression> filter);
+
+ /// \brief Sets whether the scan should be case-sensitive.
+ /// \param case_sensitive Whether the scan is case-sensitive.
+ /// /return Reference to the builder.
+ TableScanBuilder& WithCaseSensitive(bool case_sensitive);
+
+ /// \brief Sets an option for the scan.
+ /// \param property The name of the option.
+ /// \param value The value of the option.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithOption(std::string property, std::string value);
+
+ /// \brief Sets an optional limit on the number of rows to scan.
+ /// \param limit Optional limit on the number of rows.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithLimit(std::optional<int64_t> limit);
+
+ /// \brief Builds and returns a TableScan instance.
+ /// \return A Result containing the TableScan or an error.
+ Result<std::unique_ptr<TableScan>> Build();
+
+ private:
+ const Table& table_; ///< Reference to the table to scan.
+ std::vector<std::string> column_names_;
+ std::optional<int64_t> snapshot_id_;
+ TableScanContext context_; ///< Context for the scan.
+};
+
+/// \brief Represents a configured scan operation on a table.
+class ICEBERG_EXPORT TableScan {
+ public:
+ virtual ~TableScan() = default;
+
+ /// \brief Constructs a TableScan with the given context and file I/O.
+ /// \param context Scan context including snapshot, schema, and filter.
+ /// \param file_io File I/O instance for reading manifests and data files.
+ TableScan(TableScanContext context, std::shared_ptr<FileIO> file_io);
Review Comment:
I just want to make a distinction between table-related and non-table data.
##########
src/iceberg/manifest_reader.h:
##########
@@ -34,6 +34,7 @@ namespace iceberg {
/// \brief Read manifest entries from a manifest file.
class ICEBERG_EXPORT ManifestReader {
public:
+ virtual ~ManifestReader() = default;
Review Comment:
IDE 提示的，我就顺便补上的
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+class ICEBERG_EXPORT ScanTask {
+ public:
+ virtual ~ScanTask() = default;
+
+ /// \brief The number of bytes that should be read by this scan task.
+ virtual int64_t size_bytes() const = 0;
+
+ /// \brief The number of files that should be read by this scan task.
+ virtual int32_t files_count() const = 0;
+
+ /// \brief The number of rows that should be read by this scan task.
+ virtual int64_t estimated_row_count() const = 0;
+};
+
+/// \brief Represents a task to scan a portion of a data file.
+class ICEBERG_EXPORT FileScanTask : public ScanTask {
+ public:
+ FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files, int64_t start,
+ int64_t length, std::shared_ptr<Expression> residual);
+
+ /// \brief The data file that should be read by this scan task.
+ const std::shared_ptr<DataFile>& data_file() const;
+
+ /// \brief The delete files that should be read by this scan task.
+ const std::vector<std::shared_ptr<DataFile>>& delete_files() const;
+
+ /// \brief The byte offset in the data file where the scan should start.
+ int64_t start() const;
+
+ /// \brief The length in bytes to scan from the start offset.
+ int64_t length() const;
+
+ /// \brief The residual expression to apply after scanning the data file.
+ const std::shared_ptr<Expression>& residual() const;
+
+ int64_t size_bytes() const override;
+ int32_t files_count() const override;
+ int64_t estimated_row_count() const override;
+
+ private:
+ std::shared_ptr<DataFile> data_file_; ///< Data file metadata.
+ std::vector<std::shared_ptr<DataFile>> delete_files_; ///< Delete files metadata.
+
+ int64_t start_; ///< Start byte offset.
+ int64_t length_; ///< Length in bytes to scan.
+
+ std::shared_ptr<Expression> residual_; ///< Residual expression to apply.
+};
+
+/// \brief Scan context holding snapshot and scan-specific metadata.
+struct TableScanContext {
+ std::shared_ptr<TableMetadata> table_metadata; ///< Table metadata.
+ std::shared_ptr<Snapshot> snapshot; ///< Snapshot to scan.
+ std::shared_ptr<Schema> projected_schema; ///< Projected schema.
+ std::shared_ptr<Expression> filter; ///< Filter expression to apply.
+ bool case_sensitive = false; ///< Whether the scan is case-sensitive.
+ std::unordered_map<std::string, std::string>
+ options; ///< Additional options for the scan.
+ std::optional<int64_t> limit; ///< Optional limit on the number of rows to scan.
+};
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table The table to scan.
+ /// \param table_metadata The metadata of the table to scan.
+ explicit TableScanBuilder(const Table& table,
Review Comment:
I cannot get the table_metadata from the Table interface, and the table interface cannot get some fields in the table_metadata.
##########
src/iceberg/snapshot.h:
##########
@@ -222,6 +222,7 @@ struct ICEBERG_EXPORT DataOperation {
/// Snapshots are created by table operations.
struct ICEBERG_EXPORT Snapshot {
static constexpr int64_t kInvalidSnapshotId = -1;
+ static constexpr int64_t kInitialSequenceNumber = 0;
Review Comment:
I didn't notice TableMetadata::kInitialSequenceNumber, maybe I should use it directly instead of redefining it. In iceberg-java, kInitialSequenceNumber is defined in TableMetadata.
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+class ICEBERG_EXPORT ScanTask {
+ public:
+ virtual ~ScanTask() = default;
+
+ /// \brief The number of bytes that should be read by this scan task.
+ virtual int64_t size_bytes() const = 0;
+
+ /// \brief The number of files that should be read by this scan task.
+ virtual int32_t files_count() const = 0;
+
+ /// \brief The number of rows that should be read by this scan task.
+ virtual int64_t estimated_row_count() const = 0;
+};
+
+/// \brief Represents a task to scan a portion of a data file.
+class ICEBERG_EXPORT FileScanTask : public ScanTask {
+ public:
+ FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files, int64_t start,
+ int64_t length, std::shared_ptr<Expression> residual);
+
+ /// \brief The data file that should be read by this scan task.
+ const std::shared_ptr<DataFile>& data_file() const;
+
+ /// \brief The delete files that should be read by this scan task.
+ const std::vector<std::shared_ptr<DataFile>>& delete_files() const;
+
+ /// \brief The byte offset in the data file where the scan should start.
+ int64_t start() const;
+
+ /// \brief The length in bytes to scan from the start offset.
+ int64_t length() const;
+
+ /// \brief The residual expression to apply after scanning the data file.
+ const std::shared_ptr<Expression>& residual() const;
+
+ int64_t size_bytes() const override;
+ int32_t files_count() const override;
+ int64_t estimated_row_count() const override;
+
+ private:
+ std::shared_ptr<DataFile> data_file_; ///< Data file metadata.
+ std::vector<std::shared_ptr<DataFile>> delete_files_; ///< Delete files metadata.
+
+ int64_t start_; ///< Start byte offset.
+ int64_t length_; ///< Length in bytes to scan.
+
+ std::shared_ptr<Expression> residual_; ///< Residual expression to apply.
+};
+
+/// \brief Scan context holding snapshot and scan-specific metadata.
+struct TableScanContext {
+ std::shared_ptr<TableMetadata> table_metadata; ///< Table metadata.
+ std::shared_ptr<Snapshot> snapshot; ///< Snapshot to scan.
+ std::shared_ptr<Schema> projected_schema; ///< Projected schema.
+ std::shared_ptr<Expression> filter; ///< Filter expression to apply.
+ bool case_sensitive = false; ///< Whether the scan is case-sensitive.
+ std::unordered_map<std::string, std::string>
+ options; ///< Additional options for the scan.
+ std::optional<int64_t> limit; ///< Optional limit on the number of rows to scan.
+};
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table The table to scan.
+ /// \param table_metadata The metadata of the table to scan.
+ explicit TableScanBuilder(const Table& table,
+ std::shared_ptr<TableMetadata> table_metadata);
+
+ /// \brief Sets the snapshot ID to scan.
+ /// \param snapshot_id The ID of the snapshot.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSnapshotId(int64_t snapshot_id);
+
+ /// \brief Selects columns to include in the scan.
+ /// \param column_names A list of column names. If empty, all columns will be selected.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithColumnNames(std::vector<std::string> column_names);
+
+ /// \brief Sets the schema to use for the scan.
+ /// \param schema The schema to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSchema(std::shared_ptr<Schema> schema);
+
+ /// \brief Applies a filter expression to the scan.
+ /// \param filter Filter expression to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithFilter(std::shared_ptr<Expression> filter);
+
+ /// \brief Sets whether the scan should be case-sensitive.
+ /// \param case_sensitive Whether the scan is case-sensitive.
+ /// /return Reference to the builder.
+ TableScanBuilder& WithCaseSensitive(bool case_sensitive);
+
+ /// \brief Sets an option for the scan.
+ /// \param property The name of the option.
+ /// \param value The value of the option.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithOption(std::string property, std::string value);
+
+ /// \brief Sets an optional limit on the number of rows to scan.
+ /// \param limit Optional limit on the number of rows.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithLimit(std::optional<int64_t> limit);
+
+ /// \brief Builds and returns a TableScan instance.
+ /// \return A Result containing the TableScan or an error.
+ Result<std::unique_ptr<TableScan>> Build();
+
+ private:
+ const Table& table_; ///< Reference to the table to scan.
+ std::vector<std::string> column_names_;
Review Comment:
I stored them directly in the TableScanContext
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "wgtmac (via GitHub)" <gi...@apache.org>.
wgtmac commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2177651637
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+/// \brief Represents a task to scan a table or a portion of it.
+class ICEBERG_EXPORT ScanTask {
+ public:
+ virtual ~ScanTask() = default;
+
+ /// \brief The number of bytes that should be read by this scan task.
+ virtual int64_t size_bytes() const = 0;
+
+ /// \brief The number of files that should be read by this scan task.
+ virtual int32_t files_count() const = 0;
+
+ /// \brief The number of rows that should be read by this scan task.
+ virtual int64_t estimated_row_count() const = 0;
+};
+
+/// \brief Represents a task to scan a portion of a data file.
+class ICEBERG_EXPORT FileScanTask : public ScanTask {
+ public:
+ FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files, int64_t start,
+ int64_t length, std::shared_ptr<Expression> residual);
+
+ /// \brief The data file that should be read by this scan task.
+ const std::shared_ptr<DataFile>& data_file() const;
+
+ /// \brief The delete files that should be read by this scan task.
+ const std::vector<std::shared_ptr<DataFile>>& delete_files() const;
+
+ /// \brief The byte offset in the data file where the scan should start.
+ int64_t start() const;
+
+ /// \brief The length in bytes to scan from the start offset.
+ int64_t length() const;
+
+ /// \brief The residual expression to apply after scanning the data file.
+ const std::shared_ptr<Expression>& residual() const;
+
+ int64_t size_bytes() const override;
+ int32_t files_count() const override;
+ int64_t estimated_row_count() const override;
+
+ private:
+ /// \brief Data file metadata.
+ std::shared_ptr<DataFile> data_file_;
+ /// \brief Delete files metadata.
+ std::vector<std::shared_ptr<DataFile>> delete_files_;
+ /// \brief Start byte offset.
+ int64_t start_;
+ /// \brief Length in bytes to scan.
+ int64_t length_;
+ /// \brief Residual expression to apply.
+ std::shared_ptr<Expression> residual_;
+};
+
+/// \brief Scan context holding snapshot and scan-specific metadata.
+struct TableScanContext {
+ /// \brief Table metadata.
+ std::shared_ptr<TableMetadata> table_metadata;
+ /// \brief Snapshot to scan.
+ std::shared_ptr<Snapshot> snapshot;
+ /// \brief Projected schema.
+ std::shared_ptr<Schema> projected_schema;
+ /// \brief Filter expression to apply.
+ std::shared_ptr<Expression> filter;
+ /// \brief Whether the scan is case-sensitive.
+ bool case_sensitive = false;
+ /// \brief Additional options for the scan.
+ std::unordered_map<std::string, std::string> options;
+ /// \brief Optional limit on the number of rows to scan.
+ std::optional<int64_t> limit;
+};
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table The table to scan.
+ /// \param table_metadata The metadata of the table to scan.
+ explicit TableScanBuilder(const Table& table,
+ std::shared_ptr<TableMetadata> table_metadata);
+
+ /// \brief Sets the snapshot ID to scan.
+ /// \param snapshot_id The ID of the snapshot.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSnapshotId(int64_t snapshot_id);
+
+ /// \brief Selects columns to include in the scan.
+ /// \param column_names A list of column names. If empty, all columns will be selected.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithColumnNames(std::vector<std::string> column_names);
+
+ /// \brief Sets the schema to use for the scan.
+ /// \param schema The schema to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSchema(std::shared_ptr<Schema> schema);
+
+ /// \brief Applies a filter expression to the scan.
+ /// \param filter Filter expression to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithFilter(std::shared_ptr<Expression> filter);
+
+ /// \brief Sets whether the scan should be case-sensitive.
+ /// \param case_sensitive Whether the scan is case-sensitive.
+ /// /return Reference to the builder.
+ TableScanBuilder& WithCaseSensitive(bool case_sensitive);
+
+ /// \brief Sets an option for the scan.
+ /// \param property The name of the option.
+ /// \param value The value of the option.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithOption(std::string property, std::string value);
+
+ /// \brief Sets an optional limit on the number of rows to scan.
+ /// \param limit Optional limit on the number of rows.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithLimit(std::optional<int64_t> limit);
+
+ /// \brief Builds and returns a TableScan instance.
+ /// \return A Result containing the TableScan or an error.
+ Result<std::unique_ptr<TableScan>> Build();
+
+ private:
+ /// \brief Reference to the table to scan.
+ const Table& table_;
+ /// \brief column names to project in the scan.
+ std::vector<std::string> column_names_;
+ /// \brief snapshot ID to scan, if specified.
+ std::optional<int64_t> snapshot_id_;
+ /// \brief Context for the scan, including snapshot, schema, and filter.
+ TableScanContext context_;
Review Comment:
In the Java version of `TableScanContext`, `column_names_` and `snapshot_id_` are also stored in it. Should we follow the same pattern? If we do this, it seems that `TableScanBuilder` is indeed a `TableScanContextBuilder`.
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+/// \brief Represents a task to scan a table or a portion of it.
+class ICEBERG_EXPORT ScanTask {
+ public:
+ virtual ~ScanTask() = default;
+
+ /// \brief The number of bytes that should be read by this scan task.
+ virtual int64_t size_bytes() const = 0;
+
+ /// \brief The number of files that should be read by this scan task.
+ virtual int32_t files_count() const = 0;
+
+ /// \brief The number of rows that should be read by this scan task.
+ virtual int64_t estimated_row_count() const = 0;
+};
+
+/// \brief Represents a task to scan a portion of a data file.
+class ICEBERG_EXPORT FileScanTask : public ScanTask {
+ public:
+ FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files, int64_t start,
+ int64_t length, std::shared_ptr<Expression> residual);
+
+ /// \brief The data file that should be read by this scan task.
+ const std::shared_ptr<DataFile>& data_file() const;
+
+ /// \brief The delete files that should be read by this scan task.
+ const std::vector<std::shared_ptr<DataFile>>& delete_files() const;
+
+ /// \brief The byte offset in the data file where the scan should start.
+ int64_t start() const;
+
+ /// \brief The length in bytes to scan from the start offset.
+ int64_t length() const;
+
+ /// \brief The residual expression to apply after scanning the data file.
+ const std::shared_ptr<Expression>& residual() const;
+
+ int64_t size_bytes() const override;
+ int32_t files_count() const override;
+ int64_t estimated_row_count() const override;
+
+ private:
+ /// \brief Data file metadata.
+ std::shared_ptr<DataFile> data_file_;
+ /// \brief Delete files metadata.
+ std::vector<std::shared_ptr<DataFile>> delete_files_;
+ /// \brief Start byte offset.
+ int64_t start_;
+ /// \brief Length in bytes to scan.
+ int64_t length_;
+ /// \brief Residual expression to apply.
+ std::shared_ptr<Expression> residual_;
+};
+
+/// \brief Scan context holding snapshot and scan-specific metadata.
+struct TableScanContext {
+ /// \brief Table metadata.
+ std::shared_ptr<TableMetadata> table_metadata;
+ /// \brief Snapshot to scan.
+ std::shared_ptr<Snapshot> snapshot;
+ /// \brief Projected schema.
+ std::shared_ptr<Schema> projected_schema;
+ /// \brief Filter expression to apply.
+ std::shared_ptr<Expression> filter;
+ /// \brief Whether the scan is case-sensitive.
+ bool case_sensitive = false;
+ /// \brief Additional options for the scan.
+ std::unordered_map<std::string, std::string> options;
+ /// \brief Optional limit on the number of rows to scan.
+ std::optional<int64_t> limit;
+};
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table The table to scan.
+ /// \param table_metadata The metadata of the table to scan.
+ explicit TableScanBuilder(const Table& table,
+ std::shared_ptr<TableMetadata> table_metadata);
+
+ /// \brief Sets the snapshot ID to scan.
+ /// \param snapshot_id The ID of the snapshot.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSnapshotId(int64_t snapshot_id);
+
+ /// \brief Selects columns to include in the scan.
+ /// \param column_names A list of column names. If empty, all columns will be selected.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithColumnNames(std::vector<std::string> column_names);
+
+ /// \brief Sets the schema to use for the scan.
+ /// \param schema The schema to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSchema(std::shared_ptr<Schema> schema);
Review Comment:
I think we don't need this. We just need schema of a specific snapshot id which can be obtained via table_metadata. Did I miss something?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "gty404 (via GitHub)" <gi...@apache.org>.
gty404 commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2188789576
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,285 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+namespace {
+/// \brief Use indexed data structures for efficient lookups
+class DeleteFileIndex {
+ public:
+ /// \brief Build the index from a list of manifest entries.
+ explicit DeleteFileIndex(const std::vector<std::unique_ptr<ManifestEntry>>& entries) {
Review Comment:
In the PyIceberg [DataScan](https://github.com/apache/iceberg-python/blob/045dd10a45cbf0b82b095b5bde47f4731a3bf88b/pyiceberg/table/__init__.py#L1926), I see that an exception is only thrown when there is an equality delete.
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,285 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+namespace {
+/// \brief Use indexed data structures for efficient lookups
+class DeleteFileIndex {
+ public:
+ /// \brief Build the index from a list of manifest entries.
+ explicit DeleteFileIndex(const std::vector<std::unique_ptr<ManifestEntry>>& entries) {
+ for (const auto& entry : entries) {
+ const int64_t seq_num =
+ entry->sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ sequence_index.emplace(seq_num, entry.get());
+ }
+ }
+
+ /// \brief Find delete files that match the sequence number of a data entry.
+ std::vector<ManifestEntry*> FindRelevantEntries(const ManifestEntry& data_entry) const {
+ std::vector<ManifestEntry*> relevant_deletes;
+
+ // Use lower_bound for efficient range search
+ auto data_sequence_number =
+ data_entry.sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ for (auto it = sequence_index.lower_bound(data_sequence_number);
+ it != sequence_index.end(); ++it) {
+ // Additional filtering logic here
+ relevant_deletes.push_back(it->second);
+ }
+
+ return relevant_deletes;
+ }
+
+ private:
+ /// \brief Index by sequence number for quick filtering
+ std::multimap<int64_t, ManifestEntry*> sequence_index;
+};
+
+/// \brief Get matched delete files for a given data entry.
+std::vector<std::shared_ptr<DataFile>> GetMatchedDeletes(
+ const ManifestEntry& data_entry, const DeleteFileIndex& delete_file_index) {
+ const auto relevant_entries = delete_file_index.FindRelevantEntries(data_entry);
+ std::vector<std::shared_ptr<DataFile>> matched_deletes;
+ if (relevant_entries.empty()) {
+ return matched_deletes;
+ }
+
+ matched_deletes.reserve(relevant_entries.size());
+ for (const auto& delete_entry : relevant_entries) {
+ // TODO(gty404): check if the delete entry contains the data entry's file path
+ matched_deletes.emplace_back(delete_entry->data_file);
Review Comment:
Sure, it will cause this issue. I will implement the filter in the later PR, thanks.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "wgtmac (via GitHub)" <gi...@apache.org>.
wgtmac commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2110719450
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/expression/expression.h"
+#include "iceberg/file_io.h"
Review Comment:
```suggestion
```
It seems that we don't need them?
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/expression/expression.h"
+#include "iceberg/file_io.h"
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table Reference to the table to scan.
+ explicit TableScanBuilder(const Table& table);
+
+ /// \brief Sets the snapshot ID to scan.
+ /// \param snapshot_id The ID of the snapshot.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSnapshotId(int64_t snapshot_id);
+
+ /// \brief Selects columns to include in the scan.
+ /// \param column_names A list of column names.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithColumnNames(const std::vector<std::string>& column_names);
+
+ /// \brief Applies a filter expression to the scan.
+ /// \param filter Filter expression to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithFilter(const std::shared_ptr<Expression>& filter);
+
+ /// \brief Builds and returns a TableScan instance.
+ /// \return A Result containing the TableScan or an error.
+ Result<std::unique_ptr<TableScan>> Build();
+
+ private:
+ const Table& table_;
+ std::vector<std::string> column_names_;
+ std::optional<int64_t> snapshot_id_;
+ std::shared_ptr<Expression> filter_;
+};
+
+/// \brief Represents a configured scan operation on a table.
+class ICEBERG_EXPORT TableScan {
+ public:
+ /// \brief Scan context holding snapshot and scan-specific metadata.
+ struct ScanContext {
+ std::shared_ptr<Snapshot> snapshot_; ///< Snapshot to scan.
+ std::shared_ptr<Schema> schema_; ///< Projected schema.
+ std::vector<int32_t> field_ids_; ///< Field IDs of selected columns.
+ std::shared_ptr<Expression> filter_; ///< Filter expression to apply.
+ };
+
+ /// \brief Constructs a TableScan with the given context and file I/O.
+ /// \param context Scan context including snapshot, schema, and filter.
+ /// \param file_io File I/O instance for reading manifests and data files.
+ TableScan(std::unique_ptr<ScanContext> context, std::shared_ptr<FileIO> file_io);
+
+ /// \brief Plans the scan tasks by resolving manifests and data files.
+ ///
+ /// Returns a list of file scan tasks if successful.
+ /// \return A Result containing scan tasks or an error.
+ Result<std::vector<std::shared_ptr<FileScanTask>>> PlanFiles() const;
+
+ private:
+ /// \brief Creates a reader for the manifest list.
+ /// \param file_path Path to the manifest list file.
+ /// \return A Result containing the reader or an error.
+ Result<std::unique_ptr<ManifestListReader>> CreateManifestListReader(
+ const std::string& file_path) const;
+
+ /// \brief Creates a reader for a manifest file.
+ /// \param file_path Path to the manifest file.
+ /// \return A Result containing the reader or an error.
+ Result<std::unique_ptr<ManifestReader>> CreateManifestReader(
+ const std::string& file_path) const;
+
+ std::unique_ptr<ScanContext> context_;
+ std::shared_ptr<FileIO> file_io_;
+};
+
+/// \brief Represents a task to scan a portion of a data file.
+struct ICEBERG_EXPORT FileScanTask {
+ std::string file_path; ///< Path to the data file.
+ uint64_t start; ///< Start byte offset.
+ uint64_t length; ///< Length in bytes to scan.
+ std::optional<uint64_t> record_count; ///< Optional number of records.
+ DataFile::Content file_content; ///< Type of file content.
+ FileFormatType file_format; ///< Format of the data file.
+ std::shared_ptr<Schema> schema; ///< Projected schema.
+ std::vector<int32_t> field_ids; ///< Field IDs to project.
Review Comment:
Why do we need this? Isn't it included in the `schema` above?
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/expression/expression.h"
+#include "iceberg/file_io.h"
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table Reference to the table to scan.
+ explicit TableScanBuilder(const Table& table);
+
+ /// \brief Sets the snapshot ID to scan.
+ /// \param snapshot_id The ID of the snapshot.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSnapshotId(int64_t snapshot_id);
+
+ /// \brief Selects columns to include in the scan.
+ /// \param column_names A list of column names.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithColumnNames(const std::vector<std::string>& column_names);
+
+ /// \brief Applies a filter expression to the scan.
+ /// \param filter Filter expression to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithFilter(const std::shared_ptr<Expression>& filter);
+
+ /// \brief Builds and returns a TableScan instance.
+ /// \return A Result containing the TableScan or an error.
+ Result<std::unique_ptr<TableScan>> Build();
+
+ private:
+ const Table& table_;
+ std::vector<std::string> column_names_;
+ std::optional<int64_t> snapshot_id_;
+ std::shared_ptr<Expression> filter_;
+};
+
+/// \brief Represents a configured scan operation on a table.
+class ICEBERG_EXPORT TableScan {
+ public:
+ /// \brief Scan context holding snapshot and scan-specific metadata.
+ struct ScanContext {
+ std::shared_ptr<Snapshot> snapshot_; ///< Snapshot to scan.
+ std::shared_ptr<Schema> schema_; ///< Projected schema.
+ std::vector<int32_t> field_ids_; ///< Field IDs of selected columns.
+ std::shared_ptr<Expression> filter_; ///< Filter expression to apply.
+ };
+
+ /// \brief Constructs a TableScan with the given context and file I/O.
+ /// \param context Scan context including snapshot, schema, and filter.
+ /// \param file_io File I/O instance for reading manifests and data files.
+ TableScan(std::unique_ptr<ScanContext> context, std::shared_ptr<FileIO> file_io);
+
+ /// \brief Plans the scan tasks by resolving manifests and data files.
+ ///
+ /// Returns a list of file scan tasks if successful.
+ /// \return A Result containing scan tasks or an error.
+ Result<std::vector<std::shared_ptr<FileScanTask>>> PlanFiles() const;
+
+ private:
+ /// \brief Creates a reader for the manifest list.
+ /// \param file_path Path to the manifest list file.
+ /// \return A Result containing the reader or an error.
+ Result<std::unique_ptr<ManifestListReader>> CreateManifestListReader(
+ const std::string& file_path) const;
+
+ /// \brief Creates a reader for a manifest file.
+ /// \param file_path Path to the manifest file.
+ /// \return A Result containing the reader or an error.
+ Result<std::unique_ptr<ManifestReader>> CreateManifestReader(
+ const std::string& file_path) const;
+
+ std::unique_ptr<ScanContext> context_;
+ std::shared_ptr<FileIO> file_io_;
+};
+
+/// \brief Represents a task to scan a portion of a data file.
+struct ICEBERG_EXPORT FileScanTask {
Review Comment:
Before defining `FileScanTask`, should we abstract it a little bit? I haven't looked at other implementations yet. From the java impl, there's a hierarchy of `ScanTask` and `FileScanTask` is a combination of `ContentScanTask` and `PartitionScanTask`.
I'm not sure if we want to take the complexity to support the java `SplittableScanTask`, `MergableScanTask`, and `ScanTaskGroup`. We can revisit these later.
##########
src/iceberg/type_fwd.h:
##########
@@ -91,6 +91,10 @@ class LocationProvider;
class SortField;
class SortOrder;
class Table;
+class TableScan;
Review Comment:
What about putting all scan related structures into a separate section for better readability?
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/expression/expression.h"
+#include "iceberg/file_io.h"
+#include "iceberg/manifest_entry.h"
Review Comment:
If the only reason to include `manifest_entry.h` is to use `enum DataFile::Content`, what about refactoring it into `enum DataFileContent` and moving it to `type_fwd.h`?
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/expression/expression.h"
+#include "iceberg/file_io.h"
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
Review Comment:
Do we need an abstraction for common parts of other scan builders? We may need more scan types for incremental, append-only, changelog, etc. If there's not much to abstract, I'm fine to keep it as is for now and refactor them later.
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/expression/expression.h"
+#include "iceberg/file_io.h"
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table Reference to the table to scan.
+ explicit TableScanBuilder(const Table& table);
+
+ /// \brief Sets the snapshot ID to scan.
+ /// \param snapshot_id The ID of the snapshot.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSnapshotId(int64_t snapshot_id);
+
+ /// \brief Selects columns to include in the scan.
Review Comment:
Add a comment to explain what will happen if we don't set any column name?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "wgtmac (via GitHub)" <gi...@apache.org>.
wgtmac commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2177702777
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+namespace {
+/// \brief Use indexed data structures for efficient lookups
+struct DeleteFileIndex {
+ /// \brief Index by sequence number for quick filtering
+ std::multimap<int64_t, ManifestEntry*> sequence_index;
+
+ /// \brief Build the index from a list of manifest entries.
+ void BuildIndex(const std::vector<std::unique_ptr<ManifestEntry>>& entries) {
+ sequence_index.clear();
+
+ for (const auto& entry : entries) {
+ const int64_t seq_num =
+ entry->sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ sequence_index.emplace(seq_num, entry.get());
+ }
+ }
+
+ /// \brief Find delete files that match the sequence number of a data entry.
+ std::vector<ManifestEntry*> FindRelevantEntries(const ManifestEntry& data_entry) const {
+ std::vector<ManifestEntry*> relevant_deletes;
+
+ // Use lower_bound for efficient range search
+ auto data_sequence_number =
+ data_entry.sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ for (auto it = sequence_index.lower_bound(data_sequence_number);
+ it != sequence_index.end(); ++it) {
+ // Additional filtering logic here
Review Comment:
What is the additional filtering logic? Did you mean to further check if the delete files can be filtered?
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+namespace {
+/// \brief Use indexed data structures for efficient lookups
+struct DeleteFileIndex {
+ /// \brief Index by sequence number for quick filtering
+ std::multimap<int64_t, ManifestEntry*> sequence_index;
+
+ /// \brief Build the index from a list of manifest entries.
+ void BuildIndex(const std::vector<std::unique_ptr<ManifestEntry>>& entries) {
+ sequence_index.clear();
+
+ for (const auto& entry : entries) {
+ const int64_t seq_num =
+ entry->sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ sequence_index.emplace(seq_num, entry.get());
+ }
+ }
+
+ /// \brief Find delete files that match the sequence number of a data entry.
+ std::vector<ManifestEntry*> FindRelevantEntries(const ManifestEntry& data_entry) const {
+ std::vector<ManifestEntry*> relevant_deletes;
+
+ // Use lower_bound for efficient range search
+ auto data_sequence_number =
+ data_entry.sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ for (auto it = sequence_index.lower_bound(data_sequence_number);
+ it != sequence_index.end(); ++it) {
+ // Additional filtering logic here
+ relevant_deletes.push_back(it->second);
+ }
+
+ return relevant_deletes;
+ }
+};
+
+/// \brief Get matched delete files for a given data entry.
+std::vector<std::shared_ptr<DataFile>> GetMatchedDeletes(
+ const ManifestEntry& data_entry, const DeleteFileIndex& delete_file_index) {
+ const auto relevant_entries = delete_file_index.FindRelevantEntries(data_entry);
+ std::vector<std::shared_ptr<DataFile>> matched_deletes;
+ if (relevant_entries.empty()) {
+ return matched_deletes;
+ }
+
+ matched_deletes.reserve(relevant_entries.size());
+ for (const auto& delete_entry : relevant_entries) {
+ // TODO(gty404): check if the delete entry contains the data entry's file path
+ matched_deletes.emplace_back(delete_entry->data_file);
+ }
+ return matched_deletes;
+}
+} // namespace
+
+// implement FileScanTask
+FileScanTask::FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files,
+ int64_t start, int64_t length,
+ std::shared_ptr<Expression> residual)
+ : data_file_(std::move(file)),
+ delete_files_(std::move(delete_files)),
+ start_(start),
+ length_(length),
+ residual_(std::move(residual)) {}
+
+const std::shared_ptr<DataFile>& FileScanTask::data_file() const { return data_file_; }
+
+const std::vector<std::shared_ptr<DataFile>>& FileScanTask::delete_files() const {
+ return delete_files_;
+}
+
+int64_t FileScanTask::start() const { return start_; }
+
+int64_t FileScanTask::length() const { return length_; }
+
+int64_t FileScanTask::size_bytes() const {
Review Comment:
```suggestion
int64_t FileScanTask::SizeBytes() const {
```
This is not trivial.
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+namespace {
+/// \brief Use indexed data structures for efficient lookups
+struct DeleteFileIndex {
+ /// \brief Index by sequence number for quick filtering
+ std::multimap<int64_t, ManifestEntry*> sequence_index;
+
+ /// \brief Build the index from a list of manifest entries.
+ void BuildIndex(const std::vector<std::unique_ptr<ManifestEntry>>& entries) {
+ sequence_index.clear();
+
+ for (const auto& entry : entries) {
+ const int64_t seq_num =
+ entry->sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ sequence_index.emplace(seq_num, entry.get());
+ }
+ }
+
+ /// \brief Find delete files that match the sequence number of a data entry.
+ std::vector<ManifestEntry*> FindRelevantEntries(const ManifestEntry& data_entry) const {
+ std::vector<ManifestEntry*> relevant_deletes;
+
+ // Use lower_bound for efficient range search
+ auto data_sequence_number =
+ data_entry.sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ for (auto it = sequence_index.lower_bound(data_sequence_number);
+ it != sequence_index.end(); ++it) {
+ // Additional filtering logic here
+ relevant_deletes.push_back(it->second);
+ }
+
+ return relevant_deletes;
+ }
+};
+
+/// \brief Get matched delete files for a given data entry.
+std::vector<std::shared_ptr<DataFile>> GetMatchedDeletes(
+ const ManifestEntry& data_entry, const DeleteFileIndex& delete_file_index) {
+ const auto relevant_entries = delete_file_index.FindRelevantEntries(data_entry);
+ std::vector<std::shared_ptr<DataFile>> matched_deletes;
+ if (relevant_entries.empty()) {
+ return matched_deletes;
+ }
+
+ matched_deletes.reserve(relevant_entries.size());
+ for (const auto& delete_entry : relevant_entries) {
+ // TODO(gty404): check if the delete entry contains the data entry's file path
+ matched_deletes.emplace_back(delete_entry->data_file);
+ }
+ return matched_deletes;
+}
+} // namespace
+
+// implement FileScanTask
+FileScanTask::FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files,
+ int64_t start, int64_t length,
+ std::shared_ptr<Expression> residual)
+ : data_file_(std::move(file)),
+ delete_files_(std::move(delete_files)),
+ start_(start),
+ length_(length),
+ residual_(std::move(residual)) {}
+
+const std::shared_ptr<DataFile>& FileScanTask::data_file() const { return data_file_; }
+
+const std::vector<std::shared_ptr<DataFile>>& FileScanTask::delete_files() const {
+ return delete_files_;
+}
+
+int64_t FileScanTask::start() const { return start_; }
+
+int64_t FileScanTask::length() const { return length_; }
+
+int64_t FileScanTask::size_bytes() const {
+ int64_t sizeInBytes = length_;
+ std::ranges::for_each(delete_files_, [&sizeInBytes](const auto& delete_file) {
+ sizeInBytes += delete_file->file_size_in_bytes;
+ });
+ return sizeInBytes;
+}
+
+int32_t FileScanTask::files_count() const {
Review Comment:
I'm not sure if we need to rename it to `FilesCount()`. @lidavidm suggestion?
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+namespace {
+/// \brief Use indexed data structures for efficient lookups
+struct DeleteFileIndex {
+ /// \brief Index by sequence number for quick filtering
+ std::multimap<int64_t, ManifestEntry*> sequence_index;
+
+ /// \brief Build the index from a list of manifest entries.
+ void BuildIndex(const std::vector<std::unique_ptr<ManifestEntry>>& entries) {
+ sequence_index.clear();
+
+ for (const auto& entry : entries) {
+ const int64_t seq_num =
+ entry->sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ sequence_index.emplace(seq_num, entry.get());
+ }
+ }
+
+ /// \brief Find delete files that match the sequence number of a data entry.
+ std::vector<ManifestEntry*> FindRelevantEntries(const ManifestEntry& data_entry) const {
+ std::vector<ManifestEntry*> relevant_deletes;
+
+ // Use lower_bound for efficient range search
+ auto data_sequence_number =
+ data_entry.sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ for (auto it = sequence_index.lower_bound(data_sequence_number);
+ it != sequence_index.end(); ++it) {
+ // Additional filtering logic here
+ relevant_deletes.push_back(it->second);
+ }
+
+ return relevant_deletes;
+ }
+};
+
+/// \brief Get matched delete files for a given data entry.
+std::vector<std::shared_ptr<DataFile>> GetMatchedDeletes(
+ const ManifestEntry& data_entry, const DeleteFileIndex& delete_file_index) {
+ const auto relevant_entries = delete_file_index.FindRelevantEntries(data_entry);
+ std::vector<std::shared_ptr<DataFile>> matched_deletes;
+ if (relevant_entries.empty()) {
+ return matched_deletes;
+ }
+
+ matched_deletes.reserve(relevant_entries.size());
+ for (const auto& delete_entry : relevant_entries) {
+ // TODO(gty404): check if the delete entry contains the data entry's file path
+ matched_deletes.emplace_back(delete_entry->data_file);
+ }
+ return matched_deletes;
+}
+} // namespace
+
+// implement FileScanTask
+FileScanTask::FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files,
+ int64_t start, int64_t length,
+ std::shared_ptr<Expression> residual)
+ : data_file_(std::move(file)),
+ delete_files_(std::move(delete_files)),
+ start_(start),
+ length_(length),
+ residual_(std::move(residual)) {}
+
+const std::shared_ptr<DataFile>& FileScanTask::data_file() const { return data_file_; }
+
+const std::vector<std::shared_ptr<DataFile>>& FileScanTask::delete_files() const {
+ return delete_files_;
+}
+
+int64_t FileScanTask::start() const { return start_; }
+
+int64_t FileScanTask::length() const { return length_; }
+
+int64_t FileScanTask::size_bytes() const {
+ int64_t sizeInBytes = length_;
+ std::ranges::for_each(delete_files_, [&sizeInBytes](const auto& delete_file) {
+ sizeInBytes += delete_file->file_size_in_bytes;
+ });
+ return sizeInBytes;
+}
+
+int32_t FileScanTask::files_count() const {
+ return static_cast<int32_t>(delete_files_.size() + 1);
+}
+
+int64_t FileScanTask::estimated_row_count() const {
Review Comment:
ditto
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+namespace {
+/// \brief Use indexed data structures for efficient lookups
+struct DeleteFileIndex {
+ /// \brief Index by sequence number for quick filtering
+ std::multimap<int64_t, ManifestEntry*> sequence_index;
+
+ /// \brief Build the index from a list of manifest entries.
+ void BuildIndex(const std::vector<std::unique_ptr<ManifestEntry>>& entries) {
+ sequence_index.clear();
Review Comment:
Is `DeleteFileIndex` supposed to be reused? Does it make sense to make `BuildIndex` private and call it from the ctor?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "wgtmac (via GitHub)" <gi...@apache.org>.
wgtmac commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2181398464
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+/// \brief An abstract scan task.
+class ICEBERG_EXPORT ScanTask {
+ public:
+ virtual ~ScanTask() = default;
+
+ /// \brief The number of bytes that should be read by this scan task.
+ virtual int64_t SizeBytes() const = 0;
+
+ /// \brief The number of files that should be read by this scan task.
+ virtual int32_t FilesCount() const = 0;
+
+ /// \brief The number of rows that should be read by this scan task.
+ virtual int64_t EstimatedRowCount() const = 0;
+};
+
+/// \brief Task representing a data file and its corresponding delete files.
+class ICEBERG_EXPORT FileScanTask : public ScanTask {
+ public:
+ FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files, int64_t start,
+ int64_t length, std::shared_ptr<Expression> residual);
+
+ /// \brief The data file that should be read by this scan task.
+ const std::shared_ptr<DataFile>& data_file() const;
+
+ /// \brief The delete files that should be read by this scan task.
+ const std::vector<std::shared_ptr<DataFile>>& delete_files() const;
+
+ /// \brief The byte offset in the data file where the scan should start.
+ int64_t start() const;
+
+ /// \brief The length in bytes to scan from the start offset.
+ int64_t length() const;
+
+ /// \brief The residual expression to apply after scanning the data file.
+ const std::shared_ptr<Expression>& residual() const;
+
+ int64_t SizeBytes() const override;
+ int32_t FilesCount() const override;
+ int64_t EstimatedRowCount() const override;
+
+ private:
+ /// \brief Data file metadata.
+ std::shared_ptr<DataFile> data_file_;
+ /// \brief Delete files metadata.
+ std::vector<std::shared_ptr<DataFile>> delete_files_;
+ /// \brief Start byte offset.
+ int64_t start_;
+ /// \brief Length in bytes to scan.
+ int64_t length_;
+ /// \brief Residual expression to apply.
+ std::shared_ptr<Expression> residual_;
+};
+
+/// \brief Scan context holding snapshot and scan-specific metadata.
+struct TableScanContext {
+ /// \brief Table metadata.
+ std::shared_ptr<TableMetadata> table_metadata;
+ /// \brief Snapshot to scan.
+ std::shared_ptr<Snapshot> snapshot;
+ /// \brief Projected schema.
+ std::shared_ptr<Schema> projected_schema;
+ /// \brief Filter expression to apply.
+ std::shared_ptr<Expression> filter;
+ /// \brief Whether the scan is case-sensitive.
+ bool case_sensitive = false;
+ /// \brief Additional options for the scan.
+ std::unordered_map<std::string, std::string> options;
+ /// \brief Optional limit on the number of rows to scan.
+ std::optional<int64_t> limit;
+};
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table_metadata The metadata of the table to scan.
+ /// \param file_io The FileIO instance for reading manifests and data files.
+ explicit TableScanBuilder(std::shared_ptr<TableMetadata> table_metadata,
+ std::shared_ptr<FileIO> file_io);
+
+ /// \brief Sets the snapshot ID to scan.
+ /// \param snapshot_id The ID of the snapshot.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSnapshotId(int64_t snapshot_id);
+
+ /// \brief Selects columns to include in the scan.
+ /// \param column_names A list of column names. If empty, all columns will be selected.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithColumnNames(std::vector<std::string> column_names);
+
+ /// \brief Sets the schema to use for the scan.
+ /// \param schema The schema to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithProjectedSchema(std::shared_ptr<Schema> schema);
+
+ /// \brief Applies a filter expression to the scan.
+ /// \param filter Filter expression to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithFilter(std::shared_ptr<Expression> filter);
+
+ /// \brief Sets whether the scan should be case-sensitive.
+ /// \param case_sensitive Whether the scan is case-sensitive.
+ /// /return Reference to the builder.
+ TableScanBuilder& WithCaseSensitive(bool case_sensitive);
+
+ /// \brief Sets an option for the scan.
+ /// \param property The name of the option.
+ /// \param value The value of the option.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithOption(std::string property, std::string value);
+
+ /// \brief Sets an optional limit on the number of rows to scan.
+ /// \param limit Optional limit on the number of rows.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithLimit(std::optional<int64_t> limit);
+
+ /// \brief Builds and returns a TableScan instance.
+ /// \return A Result containing the TableScan or an error.
+ Result<std::unique_ptr<TableScan>> Build();
+
+ private:
+ /// \brief the file I/O instance for reading manifests and data files.
+ std::shared_ptr<FileIO> file_io_;
+ /// \brief column names to project in the scan.
+ std::vector<std::string> column_names_;
+ /// \brief snapshot ID to scan, if specified.
+ std::optional<int64_t> snapshot_id_;
+ /// \brief Context for the scan, including snapshot, schema, and filter.
+ TableScanContext context_;
+};
+
+/// \brief Represents a configured scan operation on a table.
+class ICEBERG_EXPORT TableScan {
+ public:
+ virtual ~TableScan() = default;
+
+ /// \brief Constructs a TableScan with the given context and file I/O.
+ /// \param context Scan context including snapshot, schema, and filter.
+ /// \param file_io File I/O instance for reading manifests and data files.
+ TableScan(TableScanContext context, std::shared_ptr<FileIO> file_io);
+
+ /// \brief Returns the snapshot being scanned.
+ /// \return A shared pointer to the snapshot.
+ const std::shared_ptr<Snapshot>& snapshot() const;
+
+ /// \brief Returns the projected schema for the scan.
+ /// \return A shared pointer to the projected schema.
+ const std::shared_ptr<Schema>& projection() const;
+
+ /// \brief Returns the scan context.
+ /// \return A reference to the TableScanContext.
+ const TableScanContext& context() const;
+
+ /// \brief Returns the file I/O instance used for reading manifests and data files.
+ /// \return A shared pointer to the FileIO instance.
+ const std::shared_ptr<FileIO>& io() const;
+
+ /// \brief Plans the scan tasks by resolving manifests and data files.
+ /// \return A Result containing scan tasks or an error.
+ virtual Result<std::vector<std::shared_ptr<FileScanTask>>> PlanFiles() const = 0;
+
+ protected:
+ /// \brief context for the scan, including snapshot, schema, and filter.
+ const TableScanContext context_;
+ /// \brief File I/O instance for reading manifests and data files.
+ std::shared_ptr<FileIO> file_io_;
+};
+
+/// \brief A scan that reads data files and applies delete files to filter rows.
+class ICEBERG_EXPORT DataScan : public TableScan {
Review Comment:
I'm a little bit confused about the name of Scan and ScanTask across different implementations. Should this be `DataTableScan` which produces `FileScanTask`? For `DataScan`, I think it should produce a group of `DataTask` which contains rows of `FileScanTask`.
Simply put:
Scan -> ScanTask
TableScan -> FileScanTask
DataScan -> DataTask
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+/// \brief Represents a task to scan a table or a portion of it.
+class ICEBERG_EXPORT ScanTask {
+ public:
+ virtual ~ScanTask() = default;
+
+ /// \brief The number of bytes that should be read by this scan task.
+ virtual int64_t size_bytes() const = 0;
+
+ /// \brief The number of files that should be read by this scan task.
+ virtual int32_t files_count() const = 0;
+
+ /// \brief The number of rows that should be read by this scan task.
+ virtual int64_t estimated_row_count() const = 0;
+};
+
+/// \brief Represents a task to scan a portion of a data file.
+class ICEBERG_EXPORT FileScanTask : public ScanTask {
Review Comment:
> Partition spec and value can be obtained from DataFile and Snapshot
That's a good point
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+/// \brief Represents a task to scan a table or a portion of it.
+class ICEBERG_EXPORT ScanTask {
+ public:
+ virtual ~ScanTask() = default;
+
+ /// \brief The number of bytes that should be read by this scan task.
+ virtual int64_t size_bytes() const = 0;
+
+ /// \brief The number of files that should be read by this scan task.
+ virtual int32_t files_count() const = 0;
+
+ /// \brief The number of rows that should be read by this scan task.
+ virtual int64_t estimated_row_count() const = 0;
+};
+
+/// \brief Represents a task to scan a portion of a data file.
+class ICEBERG_EXPORT FileScanTask : public ScanTask {
+ public:
+ FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files, int64_t start,
+ int64_t length, std::shared_ptr<Expression> residual);
+
+ /// \brief The data file that should be read by this scan task.
+ const std::shared_ptr<DataFile>& data_file() const;
+
+ /// \brief The delete files that should be read by this scan task.
+ const std::vector<std::shared_ptr<DataFile>>& delete_files() const;
+
+ /// \brief The byte offset in the data file where the scan should start.
+ int64_t start() const;
+
+ /// \brief The length in bytes to scan from the start offset.
+ int64_t length() const;
+
+ /// \brief The residual expression to apply after scanning the data file.
+ const std::shared_ptr<Expression>& residual() const;
+
+ int64_t size_bytes() const override;
+ int32_t files_count() const override;
+ int64_t estimated_row_count() const override;
+
+ private:
+ /// \brief Data file metadata.
+ std::shared_ptr<DataFile> data_file_;
+ /// \brief Delete files metadata.
+ std::vector<std::shared_ptr<DataFile>> delete_files_;
+ /// \brief Start byte offset.
+ int64_t start_;
+ /// \brief Length in bytes to scan.
+ int64_t length_;
+ /// \brief Residual expression to apply.
+ std::shared_ptr<Expression> residual_;
+};
+
+/// \brief Scan context holding snapshot and scan-specific metadata.
+struct TableScanContext {
+ /// \brief Table metadata.
+ std::shared_ptr<TableMetadata> table_metadata;
+ /// \brief Snapshot to scan.
+ std::shared_ptr<Snapshot> snapshot;
+ /// \brief Projected schema.
+ std::shared_ptr<Schema> projected_schema;
+ /// \brief Filter expression to apply.
+ std::shared_ptr<Expression> filter;
+ /// \brief Whether the scan is case-sensitive.
+ bool case_sensitive = false;
+ /// \brief Additional options for the scan.
+ std::unordered_map<std::string, std::string> options;
+ /// \brief Optional limit on the number of rows to scan.
+ std::optional<int64_t> limit;
+};
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table The table to scan.
+ /// \param table_metadata The metadata of the table to scan.
+ explicit TableScanBuilder(const Table& table,
+ std::shared_ptr<TableMetadata> table_metadata);
+
+ /// \brief Sets the snapshot ID to scan.
+ /// \param snapshot_id The ID of the snapshot.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSnapshotId(int64_t snapshot_id);
+
+ /// \brief Selects columns to include in the scan.
+ /// \param column_names A list of column names. If empty, all columns will be selected.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithColumnNames(std::vector<std::string> column_names);
+
+ /// \brief Sets the schema to use for the scan.
+ /// \param schema The schema to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSchema(std::shared_ptr<Schema> schema);
+
+ /// \brief Applies a filter expression to the scan.
+ /// \param filter Filter expression to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithFilter(std::shared_ptr<Expression> filter);
+
+ /// \brief Sets whether the scan should be case-sensitive.
+ /// \param case_sensitive Whether the scan is case-sensitive.
+ /// /return Reference to the builder.
+ TableScanBuilder& WithCaseSensitive(bool case_sensitive);
+
+ /// \brief Sets an option for the scan.
+ /// \param property The name of the option.
+ /// \param value The value of the option.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithOption(std::string property, std::string value);
+
+ /// \brief Sets an optional limit on the number of rows to scan.
+ /// \param limit Optional limit on the number of rows.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithLimit(std::optional<int64_t> limit);
+
+ /// \brief Builds and returns a TableScan instance.
+ /// \return A Result containing the TableScan or an error.
+ Result<std::unique_ptr<TableScan>> Build();
+
+ private:
+ /// \brief Reference to the table to scan.
+ const Table& table_;
+ /// \brief column names to project in the scan.
+ std::vector<std::string> column_names_;
+ /// \brief snapshot ID to scan, if specified.
+ std::optional<int64_t> snapshot_id_;
+ /// \brief Context for the scan, including snapshot, schema, and filter.
+ TableScanContext context_;
Review Comment:
Do you mean you will remove `TableScanContext`? It depends on whether you will reuse it to plan files for metadata tables.
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+/// \brief An abstract scan task.
+class ICEBERG_EXPORT ScanTask {
+ public:
+ virtual ~ScanTask() = default;
+
+ /// \brief The number of bytes that should be read by this scan task.
+ virtual int64_t SizeBytes() const = 0;
+
+ /// \brief The number of files that should be read by this scan task.
+ virtual int32_t FilesCount() const = 0;
+
+ /// \brief The number of rows that should be read by this scan task.
+ virtual int64_t EstimatedRowCount() const = 0;
+};
+
+/// \brief Task representing a data file and its corresponding delete files.
+class ICEBERG_EXPORT FileScanTask : public ScanTask {
+ public:
+ FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files, int64_t start,
+ int64_t length, std::shared_ptr<Expression> residual);
+
+ /// \brief The data file that should be read by this scan task.
+ const std::shared_ptr<DataFile>& data_file() const;
+
+ /// \brief The delete files that should be read by this scan task.
+ const std::vector<std::shared_ptr<DataFile>>& delete_files() const;
+
+ /// \brief The byte offset in the data file where the scan should start.
+ int64_t start() const;
+
+ /// \brief The length in bytes to scan from the start offset.
+ int64_t length() const;
+
+ /// \brief The residual expression to apply after scanning the data file.
+ const std::shared_ptr<Expression>& residual() const;
+
+ int64_t SizeBytes() const override;
+ int32_t FilesCount() const override;
+ int64_t EstimatedRowCount() const override;
+
+ private:
+ /// \brief Data file metadata.
+ std::shared_ptr<DataFile> data_file_;
+ /// \brief Delete files metadata.
+ std::vector<std::shared_ptr<DataFile>> delete_files_;
+ /// \brief Start byte offset.
+ int64_t start_;
+ /// \brief Length in bytes to scan.
+ int64_t length_;
+ /// \brief Residual expression to apply.
+ std::shared_ptr<Expression> residual_;
+};
+
+/// \brief Scan context holding snapshot and scan-specific metadata.
+struct TableScanContext {
+ /// \brief Table metadata.
+ std::shared_ptr<TableMetadata> table_metadata;
+ /// \brief Snapshot to scan.
+ std::shared_ptr<Snapshot> snapshot;
+ /// \brief Projected schema.
+ std::shared_ptr<Schema> projected_schema;
+ /// \brief Filter expression to apply.
+ std::shared_ptr<Expression> filter;
+ /// \brief Whether the scan is case-sensitive.
+ bool case_sensitive = false;
+ /// \brief Additional options for the scan.
+ std::unordered_map<std::string, std::string> options;
+ /// \brief Optional limit on the number of rows to scan.
+ std::optional<int64_t> limit;
+};
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table_metadata The metadata of the table to scan.
+ /// \param file_io The FileIO instance for reading manifests and data files.
+ explicit TableScanBuilder(std::shared_ptr<TableMetadata> table_metadata,
+ std::shared_ptr<FileIO> file_io);
+
+ /// \brief Sets the snapshot ID to scan.
+ /// \param snapshot_id The ID of the snapshot.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSnapshotId(int64_t snapshot_id);
+
+ /// \brief Selects columns to include in the scan.
+ /// \param column_names A list of column names. If empty, all columns will be selected.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithColumnNames(std::vector<std::string> column_names);
+
+ /// \brief Sets the schema to use for the scan.
+ /// \param schema The schema to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithProjectedSchema(std::shared_ptr<Schema> schema);
+
+ /// \brief Applies a filter expression to the scan.
+ /// \param filter Filter expression to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithFilter(std::shared_ptr<Expression> filter);
+
+ /// \brief Sets whether the scan should be case-sensitive.
+ /// \param case_sensitive Whether the scan is case-sensitive.
+ /// /return Reference to the builder.
+ TableScanBuilder& WithCaseSensitive(bool case_sensitive);
+
+ /// \brief Sets an option for the scan.
+ /// \param property The name of the option.
+ /// \param value The value of the option.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithOption(std::string property, std::string value);
+
+ /// \brief Sets an optional limit on the number of rows to scan.
+ /// \param limit Optional limit on the number of rows.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithLimit(std::optional<int64_t> limit);
+
+ /// \brief Builds and returns a TableScan instance.
+ /// \return A Result containing the TableScan or an error.
+ Result<std::unique_ptr<TableScan>> Build();
+
+ private:
+ /// \brief the file I/O instance for reading manifests and data files.
+ std::shared_ptr<FileIO> file_io_;
+ /// \brief column names to project in the scan.
+ std::vector<std::string> column_names_;
+ /// \brief snapshot ID to scan, if specified.
+ std::optional<int64_t> snapshot_id_;
+ /// \brief Context for the scan, including snapshot, schema, and filter.
+ TableScanContext context_;
+};
+
+/// \brief Represents a configured scan operation on a table.
+class ICEBERG_EXPORT TableScan {
+ public:
+ virtual ~TableScan() = default;
+
+ /// \brief Constructs a TableScan with the given context and file I/O.
+ /// \param context Scan context including snapshot, schema, and filter.
+ /// \param file_io File I/O instance for reading manifests and data files.
+ TableScan(TableScanContext context, std::shared_ptr<FileIO> file_io);
+
+ /// \brief Returns the snapshot being scanned.
+ /// \return A shared pointer to the snapshot.
+ const std::shared_ptr<Snapshot>& snapshot() const;
+
+ /// \brief Returns the projected schema for the scan.
+ /// \return A shared pointer to the projected schema.
+ const std::shared_ptr<Schema>& projection() const;
+
+ /// \brief Returns the scan context.
+ /// \return A reference to the TableScanContext.
+ const TableScanContext& context() const;
+
+ /// \brief Returns the file I/O instance used for reading manifests and data files.
+ /// \return A shared pointer to the FileIO instance.
+ const std::shared_ptr<FileIO>& io() const;
+
+ /// \brief Plans the scan tasks by resolving manifests and data files.
+ /// \return A Result containing scan tasks or an error.
+ virtual Result<std::vector<std::shared_ptr<FileScanTask>>> PlanFiles() const = 0;
Review Comment:
```suggestion
virtual Result<std::vector<std::shared_ptr<ScanTask>>> PlanTasks() const = 0;
```
I remember that @lishuxu has commented to rename it to `PlanTasks`. Do we need to modify the signature as above?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2182145745
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+/// \brief An abstract scan task.
+class ICEBERG_EXPORT ScanTask {
+ public:
+ virtual ~ScanTask() = default;
+
+ /// \brief The number of bytes that should be read by this scan task.
+ virtual int64_t SizeBytes() const = 0;
+
+ /// \brief The number of files that should be read by this scan task.
+ virtual int32_t FilesCount() const = 0;
+
+ /// \brief The number of rows that should be read by this scan task.
+ virtual int64_t EstimatedRowCount() const = 0;
+};
+
+/// \brief Task representing a data file and its corresponding delete files.
+class ICEBERG_EXPORT FileScanTask : public ScanTask {
+ public:
+ FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files, int64_t start,
+ int64_t length, std::shared_ptr<Expression> residual);
+
+ /// \brief The data file that should be read by this scan task.
+ const std::shared_ptr<DataFile>& data_file() const;
+
+ /// \brief The delete files that should be read by this scan task.
+ const std::vector<std::shared_ptr<DataFile>>& delete_files() const;
+
+ /// \brief The byte offset in the data file where the scan should start.
+ int64_t start() const;
+
+ /// \brief The length in bytes to scan from the start offset.
+ int64_t length() const;
+
+ /// \brief The residual expression to apply after scanning the data file.
+ const std::shared_ptr<Expression>& residual() const;
+
+ int64_t SizeBytes() const override;
+ int32_t FilesCount() const override;
+ int64_t EstimatedRowCount() const override;
+
+ private:
+ /// \brief Data file metadata.
+ std::shared_ptr<DataFile> data_file_;
+ /// \brief Delete files metadata.
+ std::vector<std::shared_ptr<DataFile>> delete_files_;
+ /// \brief Start byte offset.
+ int64_t start_;
+ /// \brief Length in bytes to scan.
+ int64_t length_;
+ /// \brief Residual expression to apply.
+ std::shared_ptr<Expression> residual_;
+};
+
+/// \brief Scan context holding snapshot and scan-specific metadata.
+struct TableScanContext {
+ /// \brief Table metadata.
+ std::shared_ptr<TableMetadata> table_metadata;
+ /// \brief Snapshot to scan.
+ std::shared_ptr<Snapshot> snapshot;
+ /// \brief Projected schema.
+ std::shared_ptr<Schema> projected_schema;
+ /// \brief Filter expression to apply.
+ std::shared_ptr<Expression> filter;
+ /// \brief Whether the scan is case-sensitive.
+ bool case_sensitive = false;
+ /// \brief Additional options for the scan.
+ std::unordered_map<std::string, std::string> options;
+ /// \brief Optional limit on the number of rows to scan.
+ std::optional<int64_t> limit;
+};
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table_metadata The metadata of the table to scan.
+ /// \param file_io The FileIO instance for reading manifests and data files.
+ explicit TableScanBuilder(std::shared_ptr<TableMetadata> table_metadata,
+ std::shared_ptr<FileIO> file_io);
+
+ /// \brief Sets the snapshot ID to scan.
+ /// \param snapshot_id The ID of the snapshot.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSnapshotId(int64_t snapshot_id);
+
+ /// \brief Selects columns to include in the scan.
+ /// \param column_names A list of column names. If empty, all columns will be selected.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithColumnNames(std::vector<std::string> column_names);
+
+ /// \brief Sets the schema to use for the scan.
+ /// \param schema The schema to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithProjectedSchema(std::shared_ptr<Schema> schema);
+
+ /// \brief Applies a filter expression to the scan.
+ /// \param filter Filter expression to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithFilter(std::shared_ptr<Expression> filter);
+
+ /// \brief Sets whether the scan should be case-sensitive.
+ /// \param case_sensitive Whether the scan is case-sensitive.
+ /// /return Reference to the builder.
+ TableScanBuilder& WithCaseSensitive(bool case_sensitive);
+
+ /// \brief Sets an option for the scan.
+ /// \param property The name of the option.
+ /// \param value The value of the option.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithOption(std::string property, std::string value);
+
+ /// \brief Sets an optional limit on the number of rows to scan.
+ /// \param limit Optional limit on the number of rows.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithLimit(std::optional<int64_t> limit);
+
+ /// \brief Builds and returns a TableScan instance.
+ /// \return A Result containing the TableScan or an error.
+ Result<std::unique_ptr<TableScan>> Build();
+
+ private:
+ /// \brief the file I/O instance for reading manifests and data files.
+ std::shared_ptr<FileIO> file_io_;
+ /// \brief column names to project in the scan.
+ std::vector<std::string> column_names_;
+ /// \brief snapshot ID to scan, if specified.
+ std::optional<int64_t> snapshot_id_;
+ /// \brief Context for the scan, including snapshot, schema, and filter.
+ TableScanContext context_;
+};
+
+/// \brief Represents a configured scan operation on a table.
+class ICEBERG_EXPORT TableScan {
+ public:
+ virtual ~TableScan() = default;
+
+ /// \brief Constructs a TableScan with the given context and file I/O.
+ /// \param context Scan context including snapshot, schema, and filter.
+ /// \param file_io File I/O instance for reading manifests and data files.
+ TableScan(TableScanContext context, std::shared_ptr<FileIO> file_io);
+
+ /// \brief Returns the snapshot being scanned.
+ /// \return A shared pointer to the snapshot.
+ const std::shared_ptr<Snapshot>& snapshot() const;
+
+ /// \brief Returns the projected schema for the scan.
+ /// \return A shared pointer to the projected schema.
+ const std::shared_ptr<Schema>& projection() const;
+
+ /// \brief Returns the scan context.
+ /// \return A reference to the TableScanContext.
+ const TableScanContext& context() const;
+
+ /// \brief Returns the file I/O instance used for reading manifests and data files.
+ /// \return A shared pointer to the FileIO instance.
+ const std::shared_ptr<FileIO>& io() const;
+
+ /// \brief Plans the scan tasks by resolving manifests and data files.
+ /// \return A Result containing scan tasks or an error.
+ virtual Result<std::vector<std::shared_ptr<FileScanTask>>> PlanFiles() const = 0;
+
+ protected:
+ /// \brief context for the scan, including snapshot, schema, and filter.
+ const TableScanContext context_;
+ /// \brief File I/O instance for reading manifests and data files.
+ std::shared_ptr<FileIO> file_io_;
+};
+
+/// \brief A scan that reads data files and applies delete files to filter rows.
+class ICEBERG_EXPORT DataScan : public TableScan {
Review Comment:
At PyIceberg we (tried to) copied the Java structure, but in the end I think it is too much OOP for Python. Maybe good to start small in C++ as well. While we can change APIs until 1.0.0, I think it is important to get this one right pretty early on, since this is the main integration point for query engines.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "wgtmac (via GitHub)" <gi...@apache.org>.
wgtmac commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2177408751
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+class ICEBERG_EXPORT ScanTask {
+ public:
+ virtual ~ScanTask() = default;
+
+ /// \brief The number of bytes that should be read by this scan task.
+ virtual int64_t size_bytes() const = 0;
+
+ /// \brief The number of files that should be read by this scan task.
+ virtual int32_t files_count() const = 0;
+
+ /// \brief The number of rows that should be read by this scan task.
+ virtual int64_t estimated_row_count() const = 0;
+};
+
+/// \brief Represents a task to scan a portion of a data file.
+class ICEBERG_EXPORT FileScanTask : public ScanTask {
+ public:
+ FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files, int64_t start,
+ int64_t length, std::shared_ptr<Expression> residual);
+
+ /// \brief The data file that should be read by this scan task.
+ const std::shared_ptr<DataFile>& data_file() const;
+
+ /// \brief The delete files that should be read by this scan task.
+ const std::vector<std::shared_ptr<DataFile>>& delete_files() const;
+
+ /// \brief The byte offset in the data file where the scan should start.
+ int64_t start() const;
+
+ /// \brief The length in bytes to scan from the start offset.
+ int64_t length() const;
+
+ /// \brief The residual expression to apply after scanning the data file.
+ const std::shared_ptr<Expression>& residual() const;
+
+ int64_t size_bytes() const override;
+ int32_t files_count() const override;
+ int64_t estimated_row_count() const override;
+
+ private:
+ std::shared_ptr<DataFile> data_file_; ///< Data file metadata.
+ std::vector<std::shared_ptr<DataFile>> delete_files_; ///< Delete files metadata.
+
+ int64_t start_; ///< Start byte offset.
+ int64_t length_; ///< Length in bytes to scan.
+
+ std::shared_ptr<Expression> residual_; ///< Residual expression to apply.
+};
+
+/// \brief Scan context holding snapshot and scan-specific metadata.
+struct TableScanContext {
+ std::shared_ptr<TableMetadata> table_metadata; ///< Table metadata.
+ std::shared_ptr<Snapshot> snapshot; ///< Snapshot to scan.
+ std::shared_ptr<Schema> projected_schema; ///< Projected schema.
+ std::shared_ptr<Expression> filter; ///< Filter expression to apply.
+ bool case_sensitive = false; ///< Whether the scan is case-sensitive.
+ std::unordered_map<std::string, std::string>
+ options; ///< Additional options for the scan.
+ std::optional<int64_t> limit; ///< Optional limit on the number of rows to scan.
+};
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table The table to scan.
+ /// \param table_metadata The metadata of the table to scan.
+ explicit TableScanBuilder(const Table& table,
Review Comment:
`Table` is just a wrapper around `TableMetadata`. I think we could either (1) expose `TableMetadata` via `Table` (and thus remove `table_metadata` from ctor), or (2) remove `table` and add `file_io` to ctor. I think passing a reference of table may cause lifetime issue. I prefer (2) here. WDYT?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "lidavidm (via GitHub)" <gi...@apache.org>.
lidavidm commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2108439814
##########
src/iceberg/result.h:
##########
@@ -31,6 +31,7 @@ namespace iceberg {
enum class ErrorKind {
kAlreadyExists,
kCommitStateUnknown,
+ kDataInvalid,
Review Comment:
Shouldn't this be `InvalidData` to be consistent with the other error codes?
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/expression/expression.h"
+#include "iceberg/file_io.h"
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table Reference to the table to scan.
+ explicit TableScanBuilder(const Table& table);
+
+ /// \brief Sets the snapshot ID to scan.
+ /// \param snapshot_id The ID of the snapshot.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSnapshotId(int64_t snapshot_id);
+
+ /// \brief Selects columns to include in the scan.
+ /// \param column_names A list of column names.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithColumnNames(const std::vector<std::string>& column_names);
+
+ /// \brief Applies a filter expression to the scan.
+ /// \param filter Filter expression to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithFilter(const std::shared_ptr<Expression>& filter);
+
+ /// \brief Builds and returns a TableScan instance.
+ /// \return A Result containing the TableScan or an error.
+ Result<std::unique_ptr<TableScan>> Build();
+
+ private:
+ const Table& table_;
+ std::vector<std::string> column_names_;
+ std::optional<int64_t> snapshot_id_;
+ std::shared_ptr<Expression> filter_;
+};
+
+/// \brief Represents a configured scan operation on a table.
+class ICEBERG_EXPORT TableScan {
+ public:
+ /// \brief Scan context holding snapshot and scan-specific metadata.
+ struct ScanContext {
+ std::shared_ptr<Snapshot> snapshot_; ///< Snapshot to scan.
+ std::shared_ptr<Schema> schema_; ///< Projected schema.
+ std::vector<int32_t> field_ids_; ///< Field IDs of selected columns.
+ std::shared_ptr<Expression> filter_; ///< Filter expression to apply.
+ };
+
+ /// \brief Constructs a TableScan with the given context and file I/O.
+ /// \param context Scan context including snapshot, schema, and filter.
+ /// \param file_io File I/O instance for reading manifests and data files.
+ TableScan(std::unique_ptr<ScanContext> context, std::shared_ptr<FileIO> file_io);
+
+ /// \brief Plans the scan tasks by resolving manifests and data files.
+ ///
+ /// Returns a list of file scan tasks if successful.
+ /// \return A Result containing scan tasks or an error.
+ Result<std::vector<std::shared_ptr<FileScanTask>>> PlanFiles() const;
+
+ private:
+ /// \brief Creates a reader for the manifest list.
+ /// \param file_path Path to the manifest list file.
+ /// \return A Result containing the reader or an error.
+ Result<std::unique_ptr<ManifestListReader>> CreateManifestListReader(
+ const std::string& file_path) const;
+
+ /// \brief Creates a reader for a manifest file.
+ /// \param file_path Path to the manifest file.
+ /// \return A Result containing the reader or an error.
+ Result<std::unique_ptr<ManifestReader>> CreateManifestReader(
+ const std::string& file_path) const;
+
+ std::unique_ptr<ScanContext> context_;
+ std::shared_ptr<FileIO> file_io_;
+};
+
+/// \brief Represents a task to scan a portion of a data file.
+struct ICEBERG_EXPORT FileScanTask {
Review Comment:
If the fields are meant to be public (because this is a struct), don't use the trailing `_` on fields
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "lishuxu (via GitHub)" <gi...@apache.org>.
lishuxu commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2174084819
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+// implement FileScanTask
+FileScanTask::FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files,
+ int64_t start, int64_t length,
+ std::shared_ptr<Expression> residual)
+ : data_file_(std::move(file)),
+ delete_files_(std::move(delete_files)),
+ start_(start),
+ length_(length),
+ residual_(std::move(residual)) {}
+
+const std::shared_ptr<DataFile>& FileScanTask::data_file() const { return data_file_; }
+
+const std::vector<std::shared_ptr<DataFile>>& FileScanTask::delete_files() const {
+ return delete_files_;
+}
+
+int64_t FileScanTask::start() const { return start_; }
+
+int64_t FileScanTask::length() const { return length_; }
+
+int64_t FileScanTask::size_bytes() const {
+ int64_t sizeInBytes = length_;
+ std::ranges::for_each(delete_files_, [&sizeInBytes](const auto& delete_file) {
+ sizeInBytes += delete_file->file_size_in_bytes;
+ });
+ return sizeInBytes;
+}
+
+int32_t FileScanTask::files_count() const {
+ return static_cast<int32_t>(delete_files_.size() + 1);
+}
+
+int64_t FileScanTask::estimated_row_count() const {
+ if (data_file_->file_size_in_bytes == 0) {
+ return 0;
+ }
+ const double scannedFileFraction =
+ static_cast<double>(length_) / data_file_->file_size_in_bytes;
+ return static_cast<int64_t>(scannedFileFraction * data_file_->record_count);
+}
+
+const std::shared_ptr<Expression>& FileScanTask::residual() const { return residual_; }
+
+TableScanBuilder::TableScanBuilder(const Table& table,
+ std::shared_ptr<TableMetadata> table_metadata)
+ : table_(table) {
+ context_.table_metadata = std::move(table_metadata);
+}
+
+TableScanBuilder& TableScanBuilder::WithColumnNames(
+ std::vector<std::string> column_names) {
+ column_names_.reserve(column_names.size());
+ column_names_ = std::move(column_names);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithSchema(std::shared_ptr<Schema> schema) {
+ context_.projected_schema = std::move(schema);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithSnapshotId(int64_t snapshot_id) {
+ snapshot_id_ = snapshot_id;
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithFilter(std::shared_ptr<Expression> filter) {
+ context_.filter = std::move(filter);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithCaseSensitive(bool case_sensitive) {
+ context_.case_sensitive = case_sensitive;
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithOption(std::string property, std::string value) {
+ context_.options[std::move(property)] = std::move(value);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithLimit(std::optional<int64_t> limit) {
+ context_.limit = limit;
+ return *this;
+}
+
+Result<std::unique_ptr<TableScan>> TableScanBuilder::Build() {
+ if (snapshot_id_) {
+ ICEBERG_ASSIGN_OR_RAISE(context_.snapshot, table_.snapshot(*snapshot_id_));
+ } else {
+ context_.snapshot = table_.current_snapshot();
+ }
+ if (context_.snapshot == nullptr) {
+ return InvalidArgument("No snapshot found for table {}", table_.name());
+ }
+
+ if (!context_.projected_schema) {
+ std::shared_ptr<Schema> schema;
+ const auto& snapshot = context_.snapshot;
+ if (snapshot->schema_id) {
+ const auto& schemas = table_.schemas();
+ if (const auto it = schemas.find(*snapshot->schema_id); it != schemas.end()) {
+ schema = it->second;
+ } else {
+ return InvalidArgument("Schema {} in snapshot {} is not found",
+ *snapshot->schema_id, snapshot->snapshot_id);
+ }
+ } else {
+ schema = table_.schema();
+ }
+
+ // TODO(gty404): collect touched columns from filter expression
+ std::vector<SchemaField> projected_fields;
+ projected_fields.reserve(column_names_.size());
Review Comment:
What should be the behavior when column_names_ is empty?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "wgtmac (via GitHub)" <gi...@apache.org>.
wgtmac commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2181413248
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+/// \brief An abstract scan task.
+class ICEBERG_EXPORT ScanTask {
+ public:
+ virtual ~ScanTask() = default;
+
+ /// \brief The number of bytes that should be read by this scan task.
+ virtual int64_t SizeBytes() const = 0;
+
+ /// \brief The number of files that should be read by this scan task.
+ virtual int32_t FilesCount() const = 0;
+
+ /// \brief The number of rows that should be read by this scan task.
+ virtual int64_t EstimatedRowCount() const = 0;
+};
+
+/// \brief Task representing a data file and its corresponding delete files.
+class ICEBERG_EXPORT FileScanTask : public ScanTask {
+ public:
+ FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files, int64_t start,
+ int64_t length, std::shared_ptr<Expression> residual);
+
+ /// \brief The data file that should be read by this scan task.
+ const std::shared_ptr<DataFile>& data_file() const;
+
+ /// \brief The delete files that should be read by this scan task.
+ const std::vector<std::shared_ptr<DataFile>>& delete_files() const;
+
+ /// \brief The byte offset in the data file where the scan should start.
+ int64_t start() const;
+
+ /// \brief The length in bytes to scan from the start offset.
+ int64_t length() const;
+
+ /// \brief The residual expression to apply after scanning the data file.
+ const std::shared_ptr<Expression>& residual() const;
+
+ int64_t SizeBytes() const override;
+ int32_t FilesCount() const override;
+ int64_t EstimatedRowCount() const override;
+
+ private:
+ /// \brief Data file metadata.
+ std::shared_ptr<DataFile> data_file_;
+ /// \brief Delete files metadata.
+ std::vector<std::shared_ptr<DataFile>> delete_files_;
+ /// \brief Start byte offset.
+ int64_t start_;
+ /// \brief Length in bytes to scan.
+ int64_t length_;
+ /// \brief Residual expression to apply.
+ std::shared_ptr<Expression> residual_;
+};
+
+/// \brief Scan context holding snapshot and scan-specific metadata.
+struct TableScanContext {
+ /// \brief Table metadata.
+ std::shared_ptr<TableMetadata> table_metadata;
+ /// \brief Snapshot to scan.
+ std::shared_ptr<Snapshot> snapshot;
+ /// \brief Projected schema.
+ std::shared_ptr<Schema> projected_schema;
+ /// \brief Filter expression to apply.
+ std::shared_ptr<Expression> filter;
+ /// \brief Whether the scan is case-sensitive.
+ bool case_sensitive = false;
+ /// \brief Additional options for the scan.
+ std::unordered_map<std::string, std::string> options;
+ /// \brief Optional limit on the number of rows to scan.
+ std::optional<int64_t> limit;
+};
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table_metadata The metadata of the table to scan.
+ /// \param file_io The FileIO instance for reading manifests and data files.
+ explicit TableScanBuilder(std::shared_ptr<TableMetadata> table_metadata,
+ std::shared_ptr<FileIO> file_io);
+
+ /// \brief Sets the snapshot ID to scan.
+ /// \param snapshot_id The ID of the snapshot.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSnapshotId(int64_t snapshot_id);
+
+ /// \brief Selects columns to include in the scan.
+ /// \param column_names A list of column names. If empty, all columns will be selected.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithColumnNames(std::vector<std::string> column_names);
+
+ /// \brief Sets the schema to use for the scan.
+ /// \param schema The schema to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithProjectedSchema(std::shared_ptr<Schema> schema);
+
+ /// \brief Applies a filter expression to the scan.
+ /// \param filter Filter expression to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithFilter(std::shared_ptr<Expression> filter);
+
+ /// \brief Sets whether the scan should be case-sensitive.
+ /// \param case_sensitive Whether the scan is case-sensitive.
+ /// /return Reference to the builder.
+ TableScanBuilder& WithCaseSensitive(bool case_sensitive);
+
+ /// \brief Sets an option for the scan.
+ /// \param property The name of the option.
+ /// \param value The value of the option.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithOption(std::string property, std::string value);
+
+ /// \brief Sets an optional limit on the number of rows to scan.
+ /// \param limit Optional limit on the number of rows.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithLimit(std::optional<int64_t> limit);
+
+ /// \brief Builds and returns a TableScan instance.
+ /// \return A Result containing the TableScan or an error.
+ Result<std::unique_ptr<TableScan>> Build();
+
+ private:
+ /// \brief the file I/O instance for reading manifests and data files.
+ std::shared_ptr<FileIO> file_io_;
+ /// \brief column names to project in the scan.
+ std::vector<std::string> column_names_;
+ /// \brief snapshot ID to scan, if specified.
+ std::optional<int64_t> snapshot_id_;
+ /// \brief Context for the scan, including snapshot, schema, and filter.
+ TableScanContext context_;
+};
+
+/// \brief Represents a configured scan operation on a table.
+class ICEBERG_EXPORT TableScan {
+ public:
+ virtual ~TableScan() = default;
+
+ /// \brief Constructs a TableScan with the given context and file I/O.
+ /// \param context Scan context including snapshot, schema, and filter.
+ /// \param file_io File I/O instance for reading manifests and data files.
+ TableScan(TableScanContext context, std::shared_ptr<FileIO> file_io);
+
+ /// \brief Returns the snapshot being scanned.
+ /// \return A shared pointer to the snapshot.
+ const std::shared_ptr<Snapshot>& snapshot() const;
+
+ /// \brief Returns the projected schema for the scan.
+ /// \return A shared pointer to the projected schema.
+ const std::shared_ptr<Schema>& projection() const;
+
+ /// \brief Returns the scan context.
+ /// \return A reference to the TableScanContext.
+ const TableScanContext& context() const;
+
+ /// \brief Returns the file I/O instance used for reading manifests and data files.
+ /// \return A shared pointer to the FileIO instance.
+ const std::shared_ptr<FileIO>& io() const;
+
+ /// \brief Plans the scan tasks by resolving manifests and data files.
+ /// \return A Result containing scan tasks or an error.
+ virtual Result<std::vector<std::shared_ptr<FileScanTask>>> PlanFiles() const = 0;
+
+ protected:
+ /// \brief context for the scan, including snapshot, schema, and filter.
+ const TableScanContext context_;
+ /// \brief File I/O instance for reading manifests and data files.
+ std::shared_ptr<FileIO> file_io_;
+};
+
+/// \brief A scan that reads data files and applies delete files to filter rows.
+class ICEBERG_EXPORT DataScan : public TableScan {
Review Comment:
BTW, I think we can constantly evolve this design because APIs can be unstable before the 1.0.0 release.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "lishuxu (via GitHub)" <gi...@apache.org>.
lishuxu commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2173784519
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+class ICEBERG_EXPORT ScanTask {
+ public:
+ virtual ~ScanTask() = default;
+
+ /// \brief The number of bytes that should be read by this scan task.
+ virtual int64_t size_bytes() const = 0;
+
+ /// \brief The number of files that should be read by this scan task.
+ virtual int32_t files_count() const = 0;
+
+ /// \brief The number of rows that should be read by this scan task.
+ virtual int64_t estimated_row_count() const = 0;
+};
+
+/// \brief Represents a task to scan a portion of a data file.
+class ICEBERG_EXPORT FileScanTask : public ScanTask {
+ public:
+ FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files, int64_t start,
+ int64_t length, std::shared_ptr<Expression> residual);
+
+ /// \brief The data file that should be read by this scan task.
+ const std::shared_ptr<DataFile>& data_file() const;
+
+ /// \brief The delete files that should be read by this scan task.
+ const std::vector<std::shared_ptr<DataFile>>& delete_files() const;
+
+ /// \brief The byte offset in the data file where the scan should start.
+ int64_t start() const;
+
+ /// \brief The length in bytes to scan from the start offset.
+ int64_t length() const;
+
+ /// \brief The residual expression to apply after scanning the data file.
+ const std::shared_ptr<Expression>& residual() const;
+
+ int64_t size_bytes() const override;
+ int32_t files_count() const override;
+ int64_t estimated_row_count() const override;
+
+ private:
+ std::shared_ptr<DataFile> data_file_; ///< Data file metadata.
+ std::vector<std::shared_ptr<DataFile>> delete_files_; ///< Delete files metadata.
+
+ int64_t start_; ///< Start byte offset.
+ int64_t length_; ///< Length in bytes to scan.
+
+ std::shared_ptr<Expression> residual_; ///< Residual expression to apply.
+};
+
+/// \brief Scan context holding snapshot and scan-specific metadata.
+struct TableScanContext {
+ std::shared_ptr<TableMetadata> table_metadata; ///< Table metadata.
+ std::shared_ptr<Snapshot> snapshot; ///< Snapshot to scan.
+ std::shared_ptr<Schema> projected_schema; ///< Projected schema.
+ std::shared_ptr<Expression> filter; ///< Filter expression to apply.
+ bool case_sensitive = false; ///< Whether the scan is case-sensitive.
+ std::unordered_map<std::string, std::string>
+ options; ///< Additional options for the scan.
+ std::optional<int64_t> limit; ///< Optional limit on the number of rows to scan.
+};
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table The table to scan.
+ /// \param table_metadata The metadata of the table to scan.
+ explicit TableScanBuilder(const Table& table,
+ std::shared_ptr<TableMetadata> table_metadata);
+
+ /// \brief Sets the snapshot ID to scan.
+ /// \param snapshot_id The ID of the snapshot.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSnapshotId(int64_t snapshot_id);
+
+ /// \brief Selects columns to include in the scan.
+ /// \param column_names A list of column names. If empty, all columns will be selected.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithColumnNames(std::vector<std::string> column_names);
+
+ /// \brief Sets the schema to use for the scan.
+ /// \param schema The schema to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSchema(std::shared_ptr<Schema> schema);
+
+ /// \brief Applies a filter expression to the scan.
+ /// \param filter Filter expression to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithFilter(std::shared_ptr<Expression> filter);
+
+ /// \brief Sets whether the scan should be case-sensitive.
+ /// \param case_sensitive Whether the scan is case-sensitive.
+ /// /return Reference to the builder.
+ TableScanBuilder& WithCaseSensitive(bool case_sensitive);
+
+ /// \brief Sets an option for the scan.
+ /// \param property The name of the option.
+ /// \param value The value of the option.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithOption(std::string property, std::string value);
+
+ /// \brief Sets an optional limit on the number of rows to scan.
+ /// \param limit Optional limit on the number of rows.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithLimit(std::optional<int64_t> limit);
+
+ /// \brief Builds and returns a TableScan instance.
+ /// \return A Result containing the TableScan or an error.
+ Result<std::unique_ptr<TableScan>> Build();
+
+ private:
+ const Table& table_; ///< Reference to the table to scan.
+ std::vector<std::string> column_names_;
+ std::optional<int64_t> snapshot_id_;
+ TableScanContext context_; ///< Context for the scan.
+};
+
+/// \brief Represents a configured scan operation on a table.
+class ICEBERG_EXPORT TableScan {
+ public:
+ virtual ~TableScan() = default;
+
+ /// \brief Constructs a TableScan with the given context and file I/O.
+ /// \param context Scan context including snapshot, schema, and filter.
+ /// \param file_io File I/O instance for reading manifests and data files.
+ TableScan(TableScanContext context, std::shared_ptr<FileIO> file_io);
+
+ /// \brief Returns the snapshot being scanned.
+ /// \return A shared pointer to the snapshot.
+ const std::shared_ptr<Snapshot>& snapshot() const;
+
+ /// \brief Returns the projected schema for the scan.
+ /// \return A shared pointer to the projected schema.
+ const std::shared_ptr<Schema>& projection() const;
+
+ /// \brief Returns the scan context.
+ /// \return A reference to the TableScanContext.
+ const TableScanContext& context() const;
+
+ /// \brief Returns the file I/O instance used for reading manifests and data files.
+ /// \return A shared pointer to the FileIO instance.
+ const std::shared_ptr<FileIO>& io() const;
+
+ /// \brief Plans the scan tasks by resolving manifests and data files.
+ /// \return A Result containing scan tasks or an error.
+ virtual Result<std::vector<std::shared_ptr<FileScanTask>>> PlanFiles() const = 0;
+
+ protected:
+ const TableScanContext context_;
+ std::shared_ptr<FileIO> file_io_;
+};
+
+class ICEBERG_EXPORT DataScan : public TableScan {
Review Comment:
Add a docstring for this class?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "wgtmac (via GitHub)" <gi...@apache.org>.
wgtmac commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2115013977
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table Reference to the table to scan.
+ explicit TableScanBuilder(const Table& table);
+
+ /// \brief Sets the snapshot ID to scan.
+ /// \param snapshot_id The ID of the snapshot.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSnapshotId(int64_t snapshot_id);
+
+ /// \brief Selects columns to include in the scan.
+ /// Defaults to none which means select all columns
+ /// \param column_names A list of column names.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithColumnNames(const std::vector<std::string>& column_names);
+
+ /// \brief Applies a filter expression to the scan.
+ /// \param filter Filter expression to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithFilter(const std::shared_ptr<Expression>& filter);
Review Comment:
```suggestion
TableScanBuilder& WithFilter(std::shared_ptr<Expression> filter);
```
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table Reference to the table to scan.
+ explicit TableScanBuilder(const Table& table);
+
+ /// \brief Sets the snapshot ID to scan.
+ /// \param snapshot_id The ID of the snapshot.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSnapshotId(int64_t snapshot_id);
+
+ /// \brief Selects columns to include in the scan.
+ /// Defaults to none which means select all columns
+ /// \param column_names A list of column names.
Review Comment:
```suggestion
/// \param column_names A list of column names. If empty, all columns will be selected.
```
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/expression/expression.h"
+#include "iceberg/file_io.h"
+#include "iceberg/manifest_entry.h"
Review Comment:
I made that change because I like shorter names. I'm fine to keep it as is for now.
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table Reference to the table to scan.
+ explicit TableScanBuilder(const Table& table);
+
+ /// \brief Sets the snapshot ID to scan.
+ /// \param snapshot_id The ID of the snapshot.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSnapshotId(int64_t snapshot_id);
+
+ /// \brief Selects columns to include in the scan.
+ /// Defaults to none which means select all columns
+ /// \param column_names A list of column names.
Review Comment:
Some thoughts on this function:
- Option for column names is much easier to use but may not be easy to select nested columns.
- The Java impl also provides a method to accept a projected `Schema` and internally these column names are converted to a `Schema` as well. Eventually we will also need a function to accept `std::shared_ptr<Schema>`.
- Similar to the Java impl, if the `filter` has used some columns as the references but are not selected here, we have to add those to projected schema as well.
##########
src/iceberg/type_fwd.h:
##########
@@ -120,11 +127,12 @@ class MetadataUpdate;
class UpdateRequirement;
class AppendFiles;
-class TableScan;
struct DataFile;
struct ManifestEntry;
struct ManifestFile;
+class ManifestReader;
Review Comment:
I was thinking that `struct`s and `class`s should be put in different sections for better alignment.
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table Reference to the table to scan.
+ explicit TableScanBuilder(const Table& table);
+
+ /// \brief Sets the snapshot ID to scan.
+ /// \param snapshot_id The ID of the snapshot.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSnapshotId(int64_t snapshot_id);
+
+ /// \brief Selects columns to include in the scan.
+ /// Defaults to none which means select all columns
+ /// \param column_names A list of column names.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithColumnNames(const std::vector<std::string>& column_names);
+
+ /// \brief Applies a filter expression to the scan.
+ /// \param filter Filter expression to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithFilter(const std::shared_ptr<Expression>& filter);
+
+ /// \brief Builds and returns a TableScan instance.
+ /// \return A Result containing the TableScan or an error.
+ Result<std::unique_ptr<TableScan>> Build();
+
+ private:
+ const Table& table_;
+ std::vector<std::string> column_names_;
+ std::optional<int64_t> snapshot_id_;
+ std::shared_ptr<Expression> filter_;
+};
+
+/// \brief Represents a configured scan operation on a table.
+class ICEBERG_EXPORT TableScan {
+ public:
+ /// \brief Scan context holding snapshot and scan-specific metadata.
+ struct ScanContext {
+ std::shared_ptr<Snapshot> snapshot; ///< Snapshot to scan.
+ std::shared_ptr<Schema> schema; ///< Table schema.
+ std::vector<int32_t> field_ids; ///< Field IDs of selected columns.
Review Comment:
Why the builder uses `column_names_` but here use `field_ids`? Should we replace it with `std::shared_ptr<Schema> projected_schema`?
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/expression/expression.h"
+#include "iceberg/file_io.h"
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table Reference to the table to scan.
+ explicit TableScanBuilder(const Table& table);
+
+ /// \brief Sets the snapshot ID to scan.
+ /// \param snapshot_id The ID of the snapshot.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSnapshotId(int64_t snapshot_id);
+
+ /// \brief Selects columns to include in the scan.
+ /// \param column_names A list of column names.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithColumnNames(const std::vector<std::string>& column_names);
+
+ /// \brief Applies a filter expression to the scan.
+ /// \param filter Filter expression to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithFilter(const std::shared_ptr<Expression>& filter);
+
+ /// \brief Builds and returns a TableScan instance.
+ /// \return A Result containing the TableScan or an error.
+ Result<std::unique_ptr<TableScan>> Build();
+
+ private:
+ const Table& table_;
+ std::vector<std::string> column_names_;
+ std::optional<int64_t> snapshot_id_;
+ std::shared_ptr<Expression> filter_;
+};
+
+/// \brief Represents a configured scan operation on a table.
+class ICEBERG_EXPORT TableScan {
+ public:
+ /// \brief Scan context holding snapshot and scan-specific metadata.
+ struct ScanContext {
+ std::shared_ptr<Snapshot> snapshot_; ///< Snapshot to scan.
+ std::shared_ptr<Schema> schema_; ///< Projected schema.
+ std::vector<int32_t> field_ids_; ///< Field IDs of selected columns.
+ std::shared_ptr<Expression> filter_; ///< Filter expression to apply.
+ };
+
+ /// \brief Constructs a TableScan with the given context and file I/O.
+ /// \param context Scan context including snapshot, schema, and filter.
+ /// \param file_io File I/O instance for reading manifests and data files.
+ TableScan(std::unique_ptr<ScanContext> context, std::shared_ptr<FileIO> file_io);
+
+ /// \brief Plans the scan tasks by resolving manifests and data files.
+ ///
+ /// Returns a list of file scan tasks if successful.
+ /// \return A Result containing scan tasks or an error.
+ Result<std::vector<std::shared_ptr<FileScanTask>>> PlanFiles() const;
+
+ private:
+ /// \brief Creates a reader for the manifest list.
+ /// \param file_path Path to the manifest list file.
+ /// \return A Result containing the reader or an error.
+ Result<std::unique_ptr<ManifestListReader>> CreateManifestListReader(
+ const std::string& file_path) const;
+
+ /// \brief Creates a reader for a manifest file.
+ /// \param file_path Path to the manifest file.
+ /// \return A Result containing the reader or an error.
+ Result<std::unique_ptr<ManifestReader>> CreateManifestReader(
+ const std::string& file_path) const;
+
+ std::unique_ptr<ScanContext> context_;
+ std::shared_ptr<FileIO> file_io_;
+};
+
+/// \brief Represents a task to scan a portion of a data file.
+struct ICEBERG_EXPORT FileScanTask {
+ std::string file_path; ///< Path to the data file.
+ uint64_t start; ///< Start byte offset.
+ uint64_t length; ///< Length in bytes to scan.
+ std::optional<uint64_t> record_count; ///< Optional number of records.
+ DataFile::Content file_content; ///< Type of file content.
+ FileFormatType file_format; ///< Format of the data file.
+ std::shared_ptr<Schema> schema; ///< Projected schema.
+ std::vector<int32_t> field_ids; ///< Field IDs to project.
Review Comment:
Isn't it too heavy to add `schema` and `field_ids` to every `FileScanTask` which shares the same context? They can be accessed via the TableScan object. I'd suggest to remove them for now until we are clear why it is needed.
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table Reference to the table to scan.
+ explicit TableScanBuilder(const Table& table);
+
+ /// \brief Sets the snapshot ID to scan.
+ /// \param snapshot_id The ID of the snapshot.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSnapshotId(int64_t snapshot_id);
+
+ /// \brief Selects columns to include in the scan.
+ /// Defaults to none which means select all columns
+ /// \param column_names A list of column names.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithColumnNames(const std::vector<std::string>& column_names);
+
+ /// \brief Applies a filter expression to the scan.
+ /// \param filter Filter expression to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithFilter(const std::shared_ptr<Expression>& filter);
+
+ /// \brief Builds and returns a TableScan instance.
+ /// \return A Result containing the TableScan or an error.
+ Result<std::unique_ptr<TableScan>> Build();
+
+ private:
+ const Table& table_;
+ std::vector<std::string> column_names_;
+ std::optional<int64_t> snapshot_id_;
+ std::shared_ptr<Expression> filter_;
+};
+
+/// \brief Represents a configured scan operation on a table.
+class ICEBERG_EXPORT TableScan {
+ public:
+ /// \brief Scan context holding snapshot and scan-specific metadata.
+ struct ScanContext {
+ std::shared_ptr<Snapshot> snapshot; ///< Snapshot to scan.
+ std::shared_ptr<Schema> schema; ///< Table schema.
+ std::vector<int32_t> field_ids; ///< Field IDs of selected columns.
+ std::shared_ptr<Expression> filter; ///< Filter expression to apply.
+ };
+
+ /// \brief Constructs a TableScan with the given context and file I/O.
+ /// \param context Scan context including snapshot, schema, and filter.
+ /// \param file_io File I/O instance for reading manifests and data files.
+ TableScan(ScanContext context, std::shared_ptr<FileIO> file_io);
+
+ /// \brief Plans the scan tasks by resolving manifests and data files.
+ ///
+ /// Returns a list of file scan tasks if successful.
+ /// \return A Result containing scan tasks or an error.
+ Result<std::vector<std::shared_ptr<FileScanTask>>> PlanFiles() const;
+
+ private:
+ /// \brief Creates a reader for the manifest list.
+ /// \param file_path Path to the manifest list file.
+ /// \return A Result containing the reader or an error.
+ Result<std::unique_ptr<ManifestListReader>> CreateManifestListReader(
Review Comment:
It seems that `CreateManifestListReader` and `CreateManifestReader` should be utility functions shared by different scan actions. What about moving them to .cc files without depending on the table scan internal state for now?
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table Reference to the table to scan.
+ explicit TableScanBuilder(const Table& table);
+
+ /// \brief Sets the snapshot ID to scan.
+ /// \param snapshot_id The ID of the snapshot.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSnapshotId(int64_t snapshot_id);
+
+ /// \brief Selects columns to include in the scan.
+ /// Defaults to none which means select all columns
+ /// \param column_names A list of column names.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithColumnNames(const std::vector<std::string>& column_names);
+
+ /// \brief Applies a filter expression to the scan.
+ /// \param filter Filter expression to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithFilter(const std::shared_ptr<Expression>& filter);
+
+ /// \brief Builds and returns a TableScan instance.
+ /// \return A Result containing the TableScan or an error.
+ Result<std::unique_ptr<TableScan>> Build();
+
+ private:
+ const Table& table_;
+ std::vector<std::string> column_names_;
+ std::optional<int64_t> snapshot_id_;
+ std::shared_ptr<Expression> filter_;
+};
+
+/// \brief Represents a configured scan operation on a table.
+class ICEBERG_EXPORT TableScan {
+ public:
+ /// \brief Scan context holding snapshot and scan-specific metadata.
+ struct ScanContext {
+ std::shared_ptr<Snapshot> snapshot; ///< Snapshot to scan.
+ std::shared_ptr<Schema> schema; ///< Table schema.
+ std::vector<int32_t> field_ids; ///< Field IDs of selected columns.
+ std::shared_ptr<Expression> filter; ///< Filter expression to apply.
+ };
+
+ /// \brief Constructs a TableScan with the given context and file I/O.
+ /// \param context Scan context including snapshot, schema, and filter.
+ /// \param file_io File I/O instance for reading manifests and data files.
+ TableScan(ScanContext context, std::shared_ptr<FileIO> file_io);
+
+ /// \brief Plans the scan tasks by resolving manifests and data files.
+ ///
+ /// Returns a list of file scan tasks if successful.
+ /// \return A Result containing scan tasks or an error.
+ Result<std::vector<std::shared_ptr<FileScanTask>>> PlanFiles() const;
+
+ private:
+ /// \brief Creates a reader for the manifest list.
+ /// \param file_path Path to the manifest list file.
+ /// \return A Result containing the reader or an error.
+ Result<std::unique_ptr<ManifestListReader>> CreateManifestListReader(
+ const std::string& file_path) const;
+
+ /// \brief Creates a reader for a manifest file.
+ /// \param file_path Path to the manifest file.
+ /// \return A Result containing the reader or an error.
+ Result<std::unique_ptr<ManifestReader>> CreateManifestReader(
+ const std::string& file_path) const;
+
+ ScanContext context_;
+ std::shared_ptr<FileIO> file_io_;
+};
+
+/// \brief Represents a task to scan a portion of a data file.
+struct ICEBERG_EXPORT FileScanTask {
+ std::string file_path; ///< Path to the data file.
Review Comment:
Should it be replaced by `DataFile`? We might need its metadata like column stats as well. Then we can remove many fields below including record_count, file_content, etc.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "gty404 (via GitHub)" <gi...@apache.org>.
gty404 commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2205110411
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+// implement FileScanTask
+FileScanTask::FileScanTask(std::shared_ptr<DataFile> file)
+ : data_file_(std::move(file)) {}
+
+const std::shared_ptr<DataFile>& FileScanTask::data_file() const { return data_file_; }
+
+int64_t FileScanTask::size_bytes() const { return data_file_->file_size_in_bytes; }
+
+int32_t FileScanTask::files_count() const { return 1; }
+
+int64_t FileScanTask::estimated_row_count() const { return data_file_->record_count; }
+
+TableScanBuilder::TableScanBuilder(std::shared_ptr<TableMetadata> table_metadata,
+ std::shared_ptr<FileIO> file_io)
+ : file_io_(std::move(file_io)) {
+ context_.table_metadata = std::move(table_metadata);
+}
+
+TableScanBuilder& TableScanBuilder::WithColumnNames(
+ std::vector<std::string> column_names) {
+ column_names_ = std::move(column_names);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithProjectedSchema(std::shared_ptr<Schema> schema) {
+ context_.projected_schema = std::move(schema);
Review Comment:
In a conflict situation, it's not very convenient to throw an exception here, I want to put the check in the build.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "gty404 (via GitHub)" <gi...@apache.org>.
gty404 commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2108542013
##########
src/iceberg/result.h:
##########
@@ -31,6 +31,7 @@ namespace iceberg {
enum class ErrorKind {
kAlreadyExists,
kCommitStateUnknown,
+ kDataInvalid,
Review Comment:
It is reasonable, I will make a modification, thanks.
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/expression/expression.h"
+#include "iceberg/file_io.h"
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table Reference to the table to scan.
+ explicit TableScanBuilder(const Table& table);
+
+ /// \brief Sets the snapshot ID to scan.
+ /// \param snapshot_id The ID of the snapshot.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSnapshotId(int64_t snapshot_id);
+
+ /// \brief Selects columns to include in the scan.
+ /// \param column_names A list of column names.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithColumnNames(const std::vector<std::string>& column_names);
+
+ /// \brief Applies a filter expression to the scan.
+ /// \param filter Filter expression to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithFilter(const std::shared_ptr<Expression>& filter);
+
+ /// \brief Builds and returns a TableScan instance.
+ /// \return A Result containing the TableScan or an error.
+ Result<std::unique_ptr<TableScan>> Build();
+
+ private:
+ const Table& table_;
+ std::vector<std::string> column_names_;
+ std::optional<int64_t> snapshot_id_;
+ std::shared_ptr<Expression> filter_;
+};
+
+/// \brief Represents a configured scan operation on a table.
+class ICEBERG_EXPORT TableScan {
+ public:
+ /// \brief Scan context holding snapshot and scan-specific metadata.
+ struct ScanContext {
+ std::shared_ptr<Snapshot> snapshot_; ///< Snapshot to scan.
+ std::shared_ptr<Schema> schema_; ///< Projected schema.
+ std::vector<int32_t> field_ids_; ///< Field IDs of selected columns.
+ std::shared_ptr<Expression> filter_; ///< Filter expression to apply.
+ };
+
+ /// \brief Constructs a TableScan with the given context and file I/O.
+ /// \param context Scan context including snapshot, schema, and filter.
+ /// \param file_io File I/O instance for reading manifests and data files.
+ TableScan(std::unique_ptr<ScanContext> context, std::shared_ptr<FileIO> file_io);
+
+ /// \brief Plans the scan tasks by resolving manifests and data files.
+ ///
+ /// Returns a list of file scan tasks if successful.
+ /// \return A Result containing scan tasks or an error.
+ Result<std::vector<std::shared_ptr<FileScanTask>>> PlanFiles() const;
+
+ private:
+ /// \brief Creates a reader for the manifest list.
+ /// \param file_path Path to the manifest list file.
+ /// \return A Result containing the reader or an error.
+ Result<std::unique_ptr<ManifestListReader>> CreateManifestListReader(
+ const std::string& file_path) const;
+
+ /// \brief Creates a reader for a manifest file.
+ /// \param file_path Path to the manifest file.
+ /// \return A Result containing the reader or an error.
+ Result<std::unique_ptr<ManifestReader>> CreateManifestReader(
+ const std::string& file_path) const;
+
+ std::unique_ptr<ScanContext> context_;
+ std::shared_ptr<FileIO> file_io_;
+};
+
+/// \brief Represents a task to scan a portion of a data file.
+struct ICEBERG_EXPORT FileScanTask {
Review Comment:
Yes, I have modified it in the subsequent commits.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "wgtmac (via GitHub)" <gi...@apache.org>.
wgtmac commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2191651049
##########
src/iceberg/table.cc:
##########
@@ -107,4 +108,8 @@ const std::vector<SnapshotLogEntry>& Table::history() const {
const std::shared_ptr<FileIO>& Table::io() const { return io_; }
+std::unique_ptr<TableScanBuilder> Table::NewScan() const {
+ return std::make_unique<TableScanBuilder>(metadata_, io_);
Review Comment:
@Fokko There are some (outdated) comments on this: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2173635835
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "wgtmac (via GitHub)" <gi...@apache.org>.
wgtmac commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2191665154
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,285 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+namespace {
+/// \brief Use indexed data structures for efficient lookups
+class DeleteFileIndex {
+ public:
+ /// \brief Build the index from a list of manifest entries.
+ explicit DeleteFileIndex(const std::vector<std::unique_ptr<ManifestEntry>>& entries) {
Review Comment:
I think for the MVP version we don't need to support delete files. Perhaps we should remove it from the current PR. In the future we can add them back with some good test cases.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "gty404 (via GitHub)" <gi...@apache.org>.
gty404 commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2177738971
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+class ICEBERG_EXPORT ScanTask {
+ public:
+ virtual ~ScanTask() = default;
+
+ /// \brief The number of bytes that should be read by this scan task.
+ virtual int64_t size_bytes() const = 0;
+
+ /// \brief The number of files that should be read by this scan task.
+ virtual int32_t files_count() const = 0;
+
+ /// \brief The number of rows that should be read by this scan task.
+ virtual int64_t estimated_row_count() const = 0;
+};
+
+/// \brief Represents a task to scan a portion of a data file.
+class ICEBERG_EXPORT FileScanTask : public ScanTask {
+ public:
+ FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files, int64_t start,
+ int64_t length, std::shared_ptr<Expression> residual);
+
+ /// \brief The data file that should be read by this scan task.
+ const std::shared_ptr<DataFile>& data_file() const;
+
+ /// \brief The delete files that should be read by this scan task.
+ const std::vector<std::shared_ptr<DataFile>>& delete_files() const;
+
+ /// \brief The byte offset in the data file where the scan should start.
+ int64_t start() const;
+
+ /// \brief The length in bytes to scan from the start offset.
+ int64_t length() const;
+
+ /// \brief The residual expression to apply after scanning the data file.
+ const std::shared_ptr<Expression>& residual() const;
+
+ int64_t size_bytes() const override;
+ int32_t files_count() const override;
+ int64_t estimated_row_count() const override;
+
+ private:
+ std::shared_ptr<DataFile> data_file_; ///< Data file metadata.
+ std::vector<std::shared_ptr<DataFile>> delete_files_; ///< Delete files metadata.
+
+ int64_t start_; ///< Start byte offset.
+ int64_t length_; ///< Length in bytes to scan.
+
+ std::shared_ptr<Expression> residual_; ///< Residual expression to apply.
+};
+
+/// \brief Scan context holding snapshot and scan-specific metadata.
+struct TableScanContext {
+ std::shared_ptr<TableMetadata> table_metadata; ///< Table metadata.
+ std::shared_ptr<Snapshot> snapshot; ///< Snapshot to scan.
+ std::shared_ptr<Schema> projected_schema; ///< Projected schema.
+ std::shared_ptr<Expression> filter; ///< Filter expression to apply.
+ bool case_sensitive = false; ///< Whether the scan is case-sensitive.
+ std::unordered_map<std::string, std::string>
+ options; ///< Additional options for the scan.
+ std::optional<int64_t> limit; ///< Optional limit on the number of rows to scan.
+};
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table The table to scan.
+ /// \param table_metadata The metadata of the table to scan.
+ explicit TableScanBuilder(const Table& table,
Review Comment:
TableMetadata should not be exposed through Table, I'll remove table then.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "gty404 (via GitHub)" <gi...@apache.org>.
gty404 commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2174083403
##########
src/iceberg/manifest_reader.h:
##########
@@ -34,6 +34,7 @@ namespace iceberg {
/// \brief Read manifest entries from a manifest file.
class ICEBERG_EXPORT ManifestReader {
public:
+ virtual ~ManifestReader() = default;
Review Comment:
The IDE prompted it, so I made it up
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "mapleFU (via GitHub)" <gi...@apache.org>.
mapleFU commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2182900887
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+namespace {
+/// \brief Use indexed data structures for efficient lookups
+class DeleteFileIndex {
+ public:
+ /// \brief Build the index from a list of manifest entries.
+ explicit DeleteFileIndex(const std::vector<std::unique_ptr<ManifestEntry>>& entries) {
+ for (const auto& entry : entries) {
+ const int64_t seq_num =
+ entry->sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ sequence_index.emplace(seq_num, entry.get());
+ }
+ }
+
+ /// \brief Find delete files that match the sequence number of a data entry.
+ std::vector<ManifestEntry*> FindRelevantEntries(const ManifestEntry& data_entry) const {
+ std::vector<ManifestEntry*> relevant_deletes;
+
+ // Use lower_bound for efficient range search
+ auto data_sequence_number =
+ data_entry.sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ for (auto it = sequence_index.lower_bound(data_sequence_number);
+ it != sequence_index.end(); ++it) {
+ // Additional filtering logic here
+ relevant_deletes.push_back(it->second);
+ }
+
+ return relevant_deletes;
+ }
+
+ private:
+ /// \brief Index by sequence number for quick filtering
+ std::multimap<int64_t, ManifestEntry*> sequence_index;
Review Comment:
```suggestion
std::multimap<int64_t, ManifestEntry*> sequence_index_;
```
For private variables? Besides, why not `std::map<int64_t, std::vector<const ManifestEntry*>>`?
##########
src/iceberg/manifest_reader.h:
##########
@@ -43,10 +44,27 @@ class ICEBERG_EXPORT ManifestReader {
/// \brief Read manifest files from a manifest list file.
class ICEBERG_EXPORT ManifestListReader {
public:
+ virtual ~ManifestListReader() = default;
virtual Result<std::span<std::unique_ptr<ManifestFile>>> Files() const = 0;
private:
std::unique_ptr<StructLikeReader> reader_;
};
+/// \brief Creates a reader for the manifest list.
+/// \param file_path Path to the manifest list file.
+/// \return A Result containing the reader or an error.
+Result<std::unique_ptr<ManifestListReader>> CreateManifestListReader(
+ const std::string_view& file_path) {
Review Comment:
Just `std::string_view` is ok. `std::string_view` is just 16B and is lightweight to construct. psABI can pass 16B in registers, so just std::string_view by value is ok
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+namespace {
+/// \brief Use indexed data structures for efficient lookups
+class DeleteFileIndex {
+ public:
+ /// \brief Build the index from a list of manifest entries.
+ explicit DeleteFileIndex(const std::vector<std::unique_ptr<ManifestEntry>>& entries) {
+ for (const auto& entry : entries) {
+ const int64_t seq_num =
+ entry->sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ sequence_index.emplace(seq_num, entry.get());
+ }
+ }
+
+ /// \brief Find delete files that match the sequence number of a data entry.
+ std::vector<ManifestEntry*> FindRelevantEntries(const ManifestEntry& data_entry) const {
+ std::vector<ManifestEntry*> relevant_deletes;
+
+ // Use lower_bound for efficient range search
+ auto data_sequence_number =
+ data_entry.sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ for (auto it = sequence_index.lower_bound(data_sequence_number);
+ it != sequence_index.end(); ++it) {
+ // Additional filtering logic here
+ relevant_deletes.push_back(it->second);
+ }
+
+ return relevant_deletes;
+ }
+
+ private:
+ /// \brief Index by sequence number for quick filtering
+ std::multimap<int64_t, ManifestEntry*> sequence_index;
+};
+
+/// \brief Get matched delete files for a given data entry.
+std::vector<std::shared_ptr<DataFile>> GetMatchedDeletes(
+ const ManifestEntry& data_entry, const DeleteFileIndex& delete_file_index) {
+ const auto relevant_entries = delete_file_index.FindRelevantEntries(data_entry);
+ std::vector<std::shared_ptr<DataFile>> matched_deletes;
+ if (relevant_entries.empty()) {
+ return matched_deletes;
+ }
+
+ matched_deletes.reserve(relevant_entries.size());
+ for (const auto& delete_entry : relevant_entries) {
+ // TODO(gty404): check if the delete entry contains the data entry's file path
+ matched_deletes.emplace_back(delete_entry->data_file);
+ }
+ return matched_deletes;
+}
+} // namespace
+
+// implement FileScanTask
+FileScanTask::FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files,
+ std::shared_ptr<Expression> residual)
+ : data_file_(std::move(file)),
+ delete_files_(std::move(delete_files)),
+ residual_(std::move(residual)) {}
+
+const std::shared_ptr<DataFile>& FileScanTask::data_file() const { return data_file_; }
+
+const std::vector<std::shared_ptr<DataFile>>& FileScanTask::delete_files() const {
+ return delete_files_;
+}
+
+int64_t FileScanTask::SizeBytes() const {
+ int64_t sizeInBytes = data_file_->file_size_in_bytes;
+ std::ranges::for_each(delete_files_, [&sizeInBytes](const auto& delete_file) {
+ sizeInBytes += delete_file->file_size_in_bytes;
+ });
+ return sizeInBytes;
+}
+
+int32_t FileScanTask::FilesCount() const {
+ return static_cast<int32_t>(delete_files_.size() + 1);
+}
+
+int64_t FileScanTask::EstimatedRowCount() const {
+ if (data_file_->file_size_in_bytes == 0) {
+ return 0;
+ }
+ const auto sizeInBytes = data_file_->file_size_in_bytes;
+ const double scannedFileFraction =
+ static_cast<double>(sizeInBytes) / data_file_->file_size_in_bytes;
+ return static_cast<int64_t>(scannedFileFraction * data_file_->record_count);
+}
+
+const std::shared_ptr<Expression>& FileScanTask::residual() const { return residual_; }
+
+TableScanBuilder::TableScanBuilder(std::shared_ptr<TableMetadata> table_metadata,
+ std::shared_ptr<FileIO> file_io)
+ : file_io_(std::move(file_io)) {
+ context_.table_metadata = std::move(table_metadata);
+}
+
+TableScanBuilder& TableScanBuilder::WithColumnNames(
+ std::vector<std::string> column_names) {
+ column_names_.reserve(column_names.size());
+ column_names_ = std::move(column_names);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithProjectedSchema(std::shared_ptr<Schema> schema) {
+ context_.projected_schema = std::move(schema);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithSnapshotId(int64_t snapshot_id) {
+ snapshot_id_ = snapshot_id;
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithFilter(std::shared_ptr<Expression> filter) {
+ context_.filter = std::move(filter);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithCaseSensitive(bool case_sensitive) {
+ context_.case_sensitive = case_sensitive;
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithOption(std::string property, std::string value) {
+ context_.options[std::move(property)] = std::move(value);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithLimit(std::optional<int64_t> limit) {
+ context_.limit = limit;
+ return *this;
+}
+
+Result<std::unique_ptr<TableScan>> TableScanBuilder::Build() {
+ const auto& table_metadata = context_.table_metadata;
+ auto snapshot_id = snapshot_id_ ? snapshot_id_ : table_metadata->current_snapshot_id;
+ if (!snapshot_id) {
+ return InvalidArgument("No snapshot ID specified for table {}",
+ table_metadata->table_uuid);
+ }
+ auto iter = std::ranges::find_if(table_metadata->snapshots,
+ [&snapshot_id](const auto& snapshot) {
+ return snapshot->snapshot_id == *snapshot_id;
+ });
+ if (iter == table_metadata->snapshots.end() || *iter == nullptr) {
+ return NotFound("Snapshot with ID {} is not found", *snapshot_id);
+ }
+ context_.snapshot = *iter;
+
+ if (!context_.projected_schema) {
+ const auto& snapshot = context_.snapshot;
+ auto schema_id =
+ snapshot->schema_id ? snapshot->schema_id : table_metadata->current_schema_id;
+ if (!schema_id) {
+ return InvalidArgument("No schema ID found in snapshot {} for table {}",
+ snapshot->snapshot_id, table_metadata->table_uuid);
+ }
+
+ const auto& schemas = table_metadata->schemas;
+ const auto it = std::ranges::find_if(schemas, [&schema_id](const auto& schema) {
+ return schema->schema_id() == *schema_id;
+ });
+ if (it == schemas.end()) {
+ return InvalidArgument("Schema {} in snapshot {} is not found",
+ *snapshot->schema_id, snapshot->snapshot_id);
+ }
+ auto schema = *it;
Review Comment:
```suggestion
const auto& schema = *it;
```
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+namespace {
+/// \brief Use indexed data structures for efficient lookups
+class DeleteFileIndex {
+ public:
+ /// \brief Build the index from a list of manifest entries.
+ explicit DeleteFileIndex(const std::vector<std::unique_ptr<ManifestEntry>>& entries) {
+ for (const auto& entry : entries) {
+ const int64_t seq_num =
+ entry->sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ sequence_index.emplace(seq_num, entry.get());
+ }
+ }
+
+ /// \brief Find delete files that match the sequence number of a data entry.
+ std::vector<ManifestEntry*> FindRelevantEntries(const ManifestEntry& data_entry) const {
+ std::vector<ManifestEntry*> relevant_deletes;
+
+ // Use lower_bound for efficient range search
+ auto data_sequence_number =
+ data_entry.sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ for (auto it = sequence_index.lower_bound(data_sequence_number);
+ it != sequence_index.end(); ++it) {
+ // Additional filtering logic here
+ relevant_deletes.push_back(it->second);
+ }
+
+ return relevant_deletes;
+ }
+
+ private:
+ /// \brief Index by sequence number for quick filtering
+ std::multimap<int64_t, ManifestEntry*> sequence_index;
+};
+
+/// \brief Get matched delete files for a given data entry.
+std::vector<std::shared_ptr<DataFile>> GetMatchedDeletes(
+ const ManifestEntry& data_entry, const DeleteFileIndex& delete_file_index) {
+ const auto relevant_entries = delete_file_index.FindRelevantEntries(data_entry);
+ std::vector<std::shared_ptr<DataFile>> matched_deletes;
+ if (relevant_entries.empty()) {
+ return matched_deletes;
+ }
+
+ matched_deletes.reserve(relevant_entries.size());
+ for (const auto& delete_entry : relevant_entries) {
+ // TODO(gty404): check if the delete entry contains the data entry's file path
+ matched_deletes.emplace_back(delete_entry->data_file);
+ }
+ return matched_deletes;
+}
+} // namespace
+
+// implement FileScanTask
+FileScanTask::FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files,
+ std::shared_ptr<Expression> residual)
+ : data_file_(std::move(file)),
+ delete_files_(std::move(delete_files)),
+ residual_(std::move(residual)) {}
+
+const std::shared_ptr<DataFile>& FileScanTask::data_file() const { return data_file_; }
+
+const std::vector<std::shared_ptr<DataFile>>& FileScanTask::delete_files() const {
+ return delete_files_;
+}
+
+int64_t FileScanTask::SizeBytes() const {
+ int64_t sizeInBytes = data_file_->file_size_in_bytes;
+ std::ranges::for_each(delete_files_, [&sizeInBytes](const auto& delete_file) {
+ sizeInBytes += delete_file->file_size_in_bytes;
+ });
+ return sizeInBytes;
+}
+
+int32_t FileScanTask::FilesCount() const {
+ return static_cast<int32_t>(delete_files_.size() + 1);
+}
+
+int64_t FileScanTask::EstimatedRowCount() const {
+ if (data_file_->file_size_in_bytes == 0) {
+ return 0;
+ }
+ const auto sizeInBytes = data_file_->file_size_in_bytes;
+ const double scannedFileFraction =
+ static_cast<double>(sizeInBytes) / data_file_->file_size_in_bytes;
+ return static_cast<int64_t>(scannedFileFraction * data_file_->record_count);
+}
+
+const std::shared_ptr<Expression>& FileScanTask::residual() const { return residual_; }
+
+TableScanBuilder::TableScanBuilder(std::shared_ptr<TableMetadata> table_metadata,
+ std::shared_ptr<FileIO> file_io)
+ : file_io_(std::move(file_io)) {
+ context_.table_metadata = std::move(table_metadata);
+}
+
+TableScanBuilder& TableScanBuilder::WithColumnNames(
+ std::vector<std::string> column_names) {
+ column_names_.reserve(column_names.size());
+ column_names_ = std::move(column_names);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithProjectedSchema(std::shared_ptr<Schema> schema) {
+ context_.projected_schema = std::move(schema);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithSnapshotId(int64_t snapshot_id) {
+ snapshot_id_ = snapshot_id;
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithFilter(std::shared_ptr<Expression> filter) {
+ context_.filter = std::move(filter);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithCaseSensitive(bool case_sensitive) {
+ context_.case_sensitive = case_sensitive;
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithOption(std::string property, std::string value) {
+ context_.options[std::move(property)] = std::move(value);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithLimit(std::optional<int64_t> limit) {
+ context_.limit = limit;
+ return *this;
+}
+
+Result<std::unique_ptr<TableScan>> TableScanBuilder::Build() {
+ const auto& table_metadata = context_.table_metadata;
+ auto snapshot_id = snapshot_id_ ? snapshot_id_ : table_metadata->current_snapshot_id;
+ if (!snapshot_id) {
+ return InvalidArgument("No snapshot ID specified for table {}",
+ table_metadata->table_uuid);
+ }
+ auto iter = std::ranges::find_if(table_metadata->snapshots,
+ [&snapshot_id](const auto& snapshot) {
+ return snapshot->snapshot_id == *snapshot_id;
+ });
+ if (iter == table_metadata->snapshots.end() || *iter == nullptr) {
+ return NotFound("Snapshot with ID {} is not found", *snapshot_id);
+ }
+ context_.snapshot = *iter;
+
+ if (!context_.projected_schema) {
+ const auto& snapshot = context_.snapshot;
+ auto schema_id =
+ snapshot->schema_id ? snapshot->schema_id : table_metadata->current_schema_id;
+ if (!schema_id) {
+ return InvalidArgument("No schema ID found in snapshot {} for table {}",
+ snapshot->snapshot_id, table_metadata->table_uuid);
+ }
+
+ const auto& schemas = table_metadata->schemas;
+ const auto it = std::ranges::find_if(schemas, [&schema_id](const auto& schema) {
+ return schema->schema_id() == *schema_id;
+ });
+ if (it == schemas.end()) {
+ return InvalidArgument("Schema {} in snapshot {} is not found",
+ *snapshot->schema_id, snapshot->snapshot_id);
+ }
+ auto schema = *it;
+
+ if (column_names_.empty()) {
+ context_.projected_schema = schema;
+ } else {
+ // TODO(gty404): collect touched columns from filter expression
+ std::vector<SchemaField> projected_fields;
+ projected_fields.reserve(column_names_.size());
+ for (const auto& column_name : column_names_) {
+ // TODO(gty404): support case-insensitive column names
+ auto field_opt = schema->GetFieldByName(column_name);
+ if (!field_opt) {
+ return InvalidArgument("Column {} not found in schema", column_name);
+ }
+ projected_fields.emplace_back(field_opt.value().get());
+ }
+ context_.projected_schema =
+ std::make_shared<Schema>(std::move(projected_fields), schema->schema_id());
+ }
+ }
+
+ return std::make_unique<DataScan>(std::move(context_), file_io_);
+}
+
+TableScan::TableScan(TableScanContext context, std::shared_ptr<FileIO> file_io)
+ : context_(std::move(context)), file_io_(std::move(file_io)) {}
+
+const std::shared_ptr<Snapshot>& TableScan::snapshot() const { return context_.snapshot; }
+
+const std::shared_ptr<Schema>& TableScan::projection() const {
+ return context_.projected_schema;
+}
+
+const TableScanContext& TableScan::context() const { return context_; }
+
+const std::shared_ptr<FileIO>& TableScan::io() const { return file_io_; }
+
+DataScan::DataScan(TableScanContext context, std::shared_ptr<FileIO> file_io)
+ : TableScan(std::move(context), std::move(file_io)) {}
+
+Result<std::vector<std::shared_ptr<FileScanTask>>> DataScan::PlanFiles() const {
+ ICEBERG_ASSIGN_OR_RAISE(auto manifest_list_reader,
+ CreateManifestListReader(context_.snapshot->manifest_list));
+ ICEBERG_ASSIGN_OR_RAISE(auto manifest_files, manifest_list_reader->Files());
+
+ std::vector<std::unique_ptr<ManifestEntry>> data_entries;
+ std::vector<std::unique_ptr<ManifestEntry>> positional_delete_entries;
+ for (const auto& manifest_file : manifest_files) {
+ ICEBERG_ASSIGN_OR_RAISE(auto manifest_reader,
+ CreateManifestReader(manifest_file->manifest_path));
+ ICEBERG_ASSIGN_OR_RAISE(auto manifests, manifest_reader->Entries());
+
+ // TODO(gty404): filter manifests using partition spec and filter expression
+
+ for (auto& manifest_entry : manifests) {
+ const auto& data_file = manifest_entry->data_file;
+ switch (data_file->content) {
+ case DataFile::Content::kData:
+ data_entries.push_back(std::move(manifest_entry));
+ break;
+ case DataFile::Content::kPositionDeletes:
+ // TODO(gty404): check if the sequence number is greater than or equal to the
+ // minimum sequence number of all manifest entries
+ positional_delete_entries.push_back(std::move(manifest_entry));
+ break;
+ case DataFile::Content::kEqualityDeletes:
+ return NotSupported("Equality deletes are not supported in data scan");
+ }
+ }
+ }
+
+ // TODO(gty404): build residual expression from filter
+ std::shared_ptr<Expression> residual;
+ std::vector<std::shared_ptr<FileScanTask>> tasks;
+ DeleteFileIndex delete_file_index(positional_delete_entries);
+ for (const auto& data_entry : data_entries) {
+ auto matched_deletes = GetMatchedDeletes(*data_entry, delete_file_index);
+ const auto& data_file = data_entry->data_file;
+ tasks.emplace_back(std::make_shared<FileScanTask>(
+ data_file, std::move(matched_deletes), std::move(residual)));
Review Comment:
```suggestion
data_file, std::move(matched_deletes), residual));
```
Seems that we cannot move `residual` because it's used multiple times in the loop?
##########
src/iceberg/manifest_reader.h:
##########
@@ -43,10 +44,27 @@ class ICEBERG_EXPORT ManifestReader {
/// \brief Read manifest files from a manifest list file.
class ICEBERG_EXPORT ManifestListReader {
public:
+ virtual ~ManifestListReader() = default;
virtual Result<std::span<std::unique_ptr<ManifestFile>>> Files() const = 0;
private:
std::unique_ptr<StructLikeReader> reader_;
};
+/// \brief Creates a reader for the manifest list.
+/// \param file_path Path to the manifest list file.
+/// \return A Result containing the reader or an error.
+Result<std::unique_ptr<ManifestListReader>> CreateManifestListReader(
+ const std::string_view& file_path) {
+ return NotImplemented("CreateManifestListReader is not implemented yet.");
+}
+
+/// \brief Creates a reader for a manifest file.
+/// \param file_path Path to the manifest file.
+/// \return A Result containing the reader or an error.
+Result<std::unique_ptr<ManifestReader>> CreateManifestReader(
+ const std::string_view& file_path) {
Review Comment:
ditto
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+namespace {
+/// \brief Use indexed data structures for efficient lookups
+class DeleteFileIndex {
+ public:
+ /// \brief Build the index from a list of manifest entries.
+ explicit DeleteFileIndex(const std::vector<std::unique_ptr<ManifestEntry>>& entries) {
+ for (const auto& entry : entries) {
+ const int64_t seq_num =
+ entry->sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ sequence_index.emplace(seq_num, entry.get());
+ }
+ }
+
+ /// \brief Find delete files that match the sequence number of a data entry.
+ std::vector<ManifestEntry*> FindRelevantEntries(const ManifestEntry& data_entry) const {
+ std::vector<ManifestEntry*> relevant_deletes;
+
+ // Use lower_bound for efficient range search
+ auto data_sequence_number =
+ data_entry.sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ for (auto it = sequence_index.lower_bound(data_sequence_number);
+ it != sequence_index.end(); ++it) {
+ // Additional filtering logic here
+ relevant_deletes.push_back(it->second);
+ }
+
+ return relevant_deletes;
+ }
+
+ private:
+ /// \brief Index by sequence number for quick filtering
+ std::multimap<int64_t, ManifestEntry*> sequence_index;
+};
+
+/// \brief Get matched delete files for a given data entry.
+std::vector<std::shared_ptr<DataFile>> GetMatchedDeletes(
+ const ManifestEntry& data_entry, const DeleteFileIndex& delete_file_index) {
+ const auto relevant_entries = delete_file_index.FindRelevantEntries(data_entry);
+ std::vector<std::shared_ptr<DataFile>> matched_deletes;
+ if (relevant_entries.empty()) {
+ return matched_deletes;
+ }
+
+ matched_deletes.reserve(relevant_entries.size());
+ for (const auto& delete_entry : relevant_entries) {
+ // TODO(gty404): check if the delete entry contains the data entry's file path
+ matched_deletes.emplace_back(delete_entry->data_file);
+ }
+ return matched_deletes;
+}
+} // namespace
+
+// implement FileScanTask
+FileScanTask::FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files,
+ std::shared_ptr<Expression> residual)
+ : data_file_(std::move(file)),
+ delete_files_(std::move(delete_files)),
+ residual_(std::move(residual)) {}
+
+const std::shared_ptr<DataFile>& FileScanTask::data_file() const { return data_file_; }
+
+const std::vector<std::shared_ptr<DataFile>>& FileScanTask::delete_files() const {
+ return delete_files_;
+}
+
+int64_t FileScanTask::SizeBytes() const {
+ int64_t sizeInBytes = data_file_->file_size_in_bytes;
+ std::ranges::for_each(delete_files_, [&sizeInBytes](const auto& delete_file) {
+ sizeInBytes += delete_file->file_size_in_bytes;
+ });
+ return sizeInBytes;
+}
+
+int32_t FileScanTask::FilesCount() const {
+ return static_cast<int32_t>(delete_files_.size() + 1);
+}
+
+int64_t FileScanTask::EstimatedRowCount() const {
+ if (data_file_->file_size_in_bytes == 0) {
+ return 0;
+ }
+ const auto sizeInBytes = data_file_->file_size_in_bytes;
+ const double scannedFileFraction =
+ static_cast<double>(sizeInBytes) / data_file_->file_size_in_bytes;
+ return static_cast<int64_t>(scannedFileFraction * data_file_->record_count);
+}
+
+const std::shared_ptr<Expression>& FileScanTask::residual() const { return residual_; }
+
+TableScanBuilder::TableScanBuilder(std::shared_ptr<TableMetadata> table_metadata,
+ std::shared_ptr<FileIO> file_io)
+ : file_io_(std::move(file_io)) {
+ context_.table_metadata = std::move(table_metadata);
+}
+
+TableScanBuilder& TableScanBuilder::WithColumnNames(
+ std::vector<std::string> column_names) {
+ column_names_.reserve(column_names.size());
+ column_names_ = std::move(column_names);
Review Comment:
We don't need reserve before move. `std::move` usally exchange memory between `column_names_` and `column_names`.
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+namespace {
+/// \brief Use indexed data structures for efficient lookups
+class DeleteFileIndex {
+ public:
+ /// \brief Build the index from a list of manifest entries.
+ explicit DeleteFileIndex(const std::vector<std::unique_ptr<ManifestEntry>>& entries) {
+ for (const auto& entry : entries) {
+ const int64_t seq_num =
+ entry->sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ sequence_index.emplace(seq_num, entry.get());
+ }
+ }
+
+ /// \brief Find delete files that match the sequence number of a data entry.
+ std::vector<ManifestEntry*> FindRelevantEntries(const ManifestEntry& data_entry) const {
+ std::vector<ManifestEntry*> relevant_deletes;
+
+ // Use lower_bound for efficient range search
+ auto data_sequence_number =
+ data_entry.sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ for (auto it = sequence_index.lower_bound(data_sequence_number);
+ it != sequence_index.end(); ++it) {
Review Comment:
Is this incorrect? Since it find the lowerbound and traverse all the sequence numbers above `data_sequence_number`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "wgtmac (via GitHub)" <gi...@apache.org>.
wgtmac commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2173636857
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+class ICEBERG_EXPORT ScanTask {
+ public:
+ virtual ~ScanTask() = default;
+
+ /// \brief The number of bytes that should be read by this scan task.
+ virtual int64_t size_bytes() const = 0;
+
+ /// \brief The number of files that should be read by this scan task.
+ virtual int32_t files_count() const = 0;
+
+ /// \brief The number of rows that should be read by this scan task.
+ virtual int64_t estimated_row_count() const = 0;
+};
+
+/// \brief Represents a task to scan a portion of a data file.
+class ICEBERG_EXPORT FileScanTask : public ScanTask {
+ public:
+ FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files, int64_t start,
+ int64_t length, std::shared_ptr<Expression> residual);
+
+ /// \brief The data file that should be read by this scan task.
+ const std::shared_ptr<DataFile>& data_file() const;
+
+ /// \brief The delete files that should be read by this scan task.
+ const std::vector<std::shared_ptr<DataFile>>& delete_files() const;
+
+ /// \brief The byte offset in the data file where the scan should start.
+ int64_t start() const;
+
+ /// \brief The length in bytes to scan from the start offset.
+ int64_t length() const;
+
+ /// \brief The residual expression to apply after scanning the data file.
+ const std::shared_ptr<Expression>& residual() const;
+
+ int64_t size_bytes() const override;
+ int32_t files_count() const override;
+ int64_t estimated_row_count() const override;
+
+ private:
+ std::shared_ptr<DataFile> data_file_; ///< Data file metadata.
+ std::vector<std::shared_ptr<DataFile>> delete_files_; ///< Delete files metadata.
+
+ int64_t start_; ///< Start byte offset.
+ int64_t length_; ///< Length in bytes to scan.
+
+ std::shared_ptr<Expression> residual_; ///< Residual expression to apply.
+};
+
+/// \brief Scan context holding snapshot and scan-specific metadata.
+struct TableScanContext {
+ std::shared_ptr<TableMetadata> table_metadata; ///< Table metadata.
+ std::shared_ptr<Snapshot> snapshot; ///< Snapshot to scan.
+ std::shared_ptr<Schema> projected_schema; ///< Projected schema.
+ std::shared_ptr<Expression> filter; ///< Filter expression to apply.
+ bool case_sensitive = false; ///< Whether the scan is case-sensitive.
+ std::unordered_map<std::string, std::string>
+ options; ///< Additional options for the scan.
+ std::optional<int64_t> limit; ///< Optional limit on the number of rows to scan.
+};
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table The table to scan.
+ /// \param table_metadata The metadata of the table to scan.
+ explicit TableScanBuilder(const Table& table,
+ std::shared_ptr<TableMetadata> table_metadata);
+
+ /// \brief Sets the snapshot ID to scan.
+ /// \param snapshot_id The ID of the snapshot.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSnapshotId(int64_t snapshot_id);
+
+ /// \brief Selects columns to include in the scan.
+ /// \param column_names A list of column names. If empty, all columns will be selected.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithColumnNames(std::vector<std::string> column_names);
+
+ /// \brief Sets the schema to use for the scan.
+ /// \param schema The schema to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSchema(std::shared_ptr<Schema> schema);
+
+ /// \brief Applies a filter expression to the scan.
+ /// \param filter Filter expression to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithFilter(std::shared_ptr<Expression> filter);
+
+ /// \brief Sets whether the scan should be case-sensitive.
+ /// \param case_sensitive Whether the scan is case-sensitive.
+ /// /return Reference to the builder.
+ TableScanBuilder& WithCaseSensitive(bool case_sensitive);
+
+ /// \brief Sets an option for the scan.
+ /// \param property The name of the option.
+ /// \param value The value of the option.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithOption(std::string property, std::string value);
+
+ /// \brief Sets an optional limit on the number of rows to scan.
+ /// \param limit Optional limit on the number of rows.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithLimit(std::optional<int64_t> limit);
+
+ /// \brief Builds and returns a TableScan instance.
+ /// \return A Result containing the TableScan or an error.
+ Result<std::unique_ptr<TableScan>> Build();
+
+ private:
+ const Table& table_; ///< Reference to the table to scan.
+ std::vector<std::string> column_names_;
+ std::optional<int64_t> snapshot_id_;
+ TableScanContext context_; ///< Context for the scan.
+};
+
+/// \brief Represents a configured scan operation on a table.
+class ICEBERG_EXPORT TableScan {
+ public:
+ virtual ~TableScan() = default;
+
+ /// \brief Constructs a TableScan with the given context and file I/O.
+ /// \param context Scan context including snapshot, schema, and filter.
+ /// \param file_io File I/O instance for reading manifests and data files.
+ TableScan(TableScanContext context, std::shared_ptr<FileIO> file_io);
Review Comment:
What is the reason that `file_io` is not included in the `context`?
##########
src/iceberg/snapshot.h:
##########
@@ -222,6 +222,7 @@ struct ICEBERG_EXPORT DataOperation {
/// Snapshots are created by table operations.
struct ICEBERG_EXPORT Snapshot {
static constexpr int64_t kInvalidSnapshotId = -1;
+ static constexpr int64_t kInitialSequenceNumber = 0;
Review Comment:
I'm not sure which one is the best place to define this. Attribute like sequence number belongs to both TableMetadata and Snapshot. Perhaps we can define a `iceberg/constants.h` to define those common constants?
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+class ICEBERG_EXPORT ScanTask {
Review Comment:
Add a docstring for this class?
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+class ICEBERG_EXPORT ScanTask {
+ public:
+ virtual ~ScanTask() = default;
+
+ /// \brief The number of bytes that should be read by this scan task.
+ virtual int64_t size_bytes() const = 0;
+
+ /// \brief The number of files that should be read by this scan task.
+ virtual int32_t files_count() const = 0;
+
+ /// \brief The number of rows that should be read by this scan task.
+ virtual int64_t estimated_row_count() const = 0;
+};
+
+/// \brief Represents a task to scan a portion of a data file.
+class ICEBERG_EXPORT FileScanTask : public ScanTask {
+ public:
+ FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files, int64_t start,
+ int64_t length, std::shared_ptr<Expression> residual);
+
+ /// \brief The data file that should be read by this scan task.
+ const std::shared_ptr<DataFile>& data_file() const;
+
+ /// \brief The delete files that should be read by this scan task.
+ const std::vector<std::shared_ptr<DataFile>>& delete_files() const;
+
+ /// \brief The byte offset in the data file where the scan should start.
+ int64_t start() const;
+
+ /// \brief The length in bytes to scan from the start offset.
+ int64_t length() const;
+
+ /// \brief The residual expression to apply after scanning the data file.
+ const std::shared_ptr<Expression>& residual() const;
+
+ int64_t size_bytes() const override;
+ int32_t files_count() const override;
+ int64_t estimated_row_count() const override;
+
+ private:
+ std::shared_ptr<DataFile> data_file_; ///< Data file metadata.
+ std::vector<std::shared_ptr<DataFile>> delete_files_; ///< Delete files metadata.
+
+ int64_t start_; ///< Start byte offset.
+ int64_t length_; ///< Length in bytes to scan.
+
+ std::shared_ptr<Expression> residual_; ///< Residual expression to apply.
+};
+
+/// \brief Scan context holding snapshot and scan-specific metadata.
+struct TableScanContext {
+ std::shared_ptr<TableMetadata> table_metadata; ///< Table metadata.
+ std::shared_ptr<Snapshot> snapshot; ///< Snapshot to scan.
+ std::shared_ptr<Schema> projected_schema; ///< Projected schema.
+ std::shared_ptr<Expression> filter; ///< Filter expression to apply.
+ bool case_sensitive = false; ///< Whether the scan is case-sensitive.
+ std::unordered_map<std::string, std::string>
+ options; ///< Additional options for the scan.
+ std::optional<int64_t> limit; ///< Optional limit on the number of rows to scan.
+};
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table The table to scan.
+ /// \param table_metadata The metadata of the table to scan.
+ explicit TableScanBuilder(const Table& table,
Review Comment:
Why do we need both `table` and `table_metadata`?
##########
src/iceberg/manifest_reader.h:
##########
@@ -43,10 +44,27 @@ class ICEBERG_EXPORT ManifestReader {
/// \brief Read manifest files from a manifest list file.
class ICEBERG_EXPORT ManifestListReader {
public:
+ virtual ~ManifestListReader() = default;
virtual Result<std::span<std::unique_ptr<ManifestFile>>> Files() const = 0;
private:
std::unique_ptr<StructLikeReader> reader_;
};
+/// \brief Creates a reader for the manifest list.
+/// \param file_path Path to the manifest list file.
+/// \return A Result containing the reader or an error.
+Result<std::unique_ptr<ManifestListReader>> CreateManifestListReader(
+ const std::string& file_path) {
Review Comment:
I think this is not enough. At least we need extra parameters like `table_format_version` and `file_io`.
##########
src/iceberg/manifest_reader.h:
##########
@@ -34,6 +34,7 @@ namespace iceberg {
/// \brief Read manifest entries from a manifest file.
class ICEBERG_EXPORT ManifestReader {
public:
+ virtual ~ManifestReader() = default;
Review Comment:
Why do we need to explicitly define it?
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+class ICEBERG_EXPORT ScanTask {
+ public:
+ virtual ~ScanTask() = default;
+
+ /// \brief The number of bytes that should be read by this scan task.
+ virtual int64_t size_bytes() const = 0;
+
+ /// \brief The number of files that should be read by this scan task.
+ virtual int32_t files_count() const = 0;
+
+ /// \brief The number of rows that should be read by this scan task.
+ virtual int64_t estimated_row_count() const = 0;
+};
+
+/// \brief Represents a task to scan a portion of a data file.
+class ICEBERG_EXPORT FileScanTask : public ScanTask {
+ public:
+ FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files, int64_t start,
+ int64_t length, std::shared_ptr<Expression> residual);
+
+ /// \brief The data file that should be read by this scan task.
+ const std::shared_ptr<DataFile>& data_file() const;
+
+ /// \brief The delete files that should be read by this scan task.
+ const std::vector<std::shared_ptr<DataFile>>& delete_files() const;
+
+ /// \brief The byte offset in the data file where the scan should start.
+ int64_t start() const;
+
+ /// \brief The length in bytes to scan from the start offset.
+ int64_t length() const;
+
+ /// \brief The residual expression to apply after scanning the data file.
+ const std::shared_ptr<Expression>& residual() const;
+
+ int64_t size_bytes() const override;
+ int32_t files_count() const override;
+ int64_t estimated_row_count() const override;
+
+ private:
+ std::shared_ptr<DataFile> data_file_; ///< Data file metadata.
+ std::vector<std::shared_ptr<DataFile>> delete_files_; ///< Delete files metadata.
Review Comment:
Should this be `DeleteFile`?
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+class ICEBERG_EXPORT ScanTask {
+ public:
+ virtual ~ScanTask() = default;
+
+ /// \brief The number of bytes that should be read by this scan task.
+ virtual int64_t size_bytes() const = 0;
+
+ /// \brief The number of files that should be read by this scan task.
+ virtual int32_t files_count() const = 0;
+
+ /// \brief The number of rows that should be read by this scan task.
+ virtual int64_t estimated_row_count() const = 0;
+};
+
+/// \brief Represents a task to scan a portion of a data file.
+class ICEBERG_EXPORT FileScanTask : public ScanTask {
+ public:
+ FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files, int64_t start,
+ int64_t length, std::shared_ptr<Expression> residual);
+
+ /// \brief The data file that should be read by this scan task.
+ const std::shared_ptr<DataFile>& data_file() const;
+
+ /// \brief The delete files that should be read by this scan task.
+ const std::vector<std::shared_ptr<DataFile>>& delete_files() const;
+
+ /// \brief The byte offset in the data file where the scan should start.
+ int64_t start() const;
+
+ /// \brief The length in bytes to scan from the start offset.
+ int64_t length() const;
+
+ /// \brief The residual expression to apply after scanning the data file.
+ const std::shared_ptr<Expression>& residual() const;
+
+ int64_t size_bytes() const override;
+ int32_t files_count() const override;
+ int64_t estimated_row_count() const override;
+
+ private:
+ std::shared_ptr<DataFile> data_file_; ///< Data file metadata.
+ std::vector<std::shared_ptr<DataFile>> delete_files_; ///< Delete files metadata.
+
+ int64_t start_; ///< Start byte offset.
+ int64_t length_; ///< Length in bytes to scan.
+
+ std::shared_ptr<Expression> residual_; ///< Residual expression to apply.
+};
+
+/// \brief Scan context holding snapshot and scan-specific metadata.
+struct TableScanContext {
+ std::shared_ptr<TableMetadata> table_metadata; ///< Table metadata.
+ std::shared_ptr<Snapshot> snapshot; ///< Snapshot to scan.
+ std::shared_ptr<Schema> projected_schema; ///< Projected schema.
+ std::shared_ptr<Expression> filter; ///< Filter expression to apply.
+ bool case_sensitive = false; ///< Whether the scan is case-sensitive.
+ std::unordered_map<std::string, std::string>
+ options; ///< Additional options for the scan.
+ std::optional<int64_t> limit; ///< Optional limit on the number of rows to scan.
+};
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table The table to scan.
+ /// \param table_metadata The metadata of the table to scan.
+ explicit TableScanBuilder(const Table& table,
+ std::shared_ptr<TableMetadata> table_metadata);
+
+ /// \brief Sets the snapshot ID to scan.
+ /// \param snapshot_id The ID of the snapshot.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSnapshotId(int64_t snapshot_id);
+
+ /// \brief Selects columns to include in the scan.
+ /// \param column_names A list of column names. If empty, all columns will be selected.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithColumnNames(std::vector<std::string> column_names);
+
+ /// \brief Sets the schema to use for the scan.
+ /// \param schema The schema to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSchema(std::shared_ptr<Schema> schema);
+
+ /// \brief Applies a filter expression to the scan.
+ /// \param filter Filter expression to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithFilter(std::shared_ptr<Expression> filter);
+
+ /// \brief Sets whether the scan should be case-sensitive.
+ /// \param case_sensitive Whether the scan is case-sensitive.
+ /// /return Reference to the builder.
+ TableScanBuilder& WithCaseSensitive(bool case_sensitive);
+
+ /// \brief Sets an option for the scan.
+ /// \param property The name of the option.
+ /// \param value The value of the option.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithOption(std::string property, std::string value);
+
+ /// \brief Sets an optional limit on the number of rows to scan.
+ /// \param limit Optional limit on the number of rows.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithLimit(std::optional<int64_t> limit);
+
+ /// \brief Builds and returns a TableScan instance.
+ /// \return A Result containing the TableScan or an error.
+ Result<std::unique_ptr<TableScan>> Build();
+
+ private:
+ const Table& table_; ///< Reference to the table to scan.
+ std::vector<std::string> column_names_;
Review Comment:
Why there is no `std::shared_ptr<Schema>` for `WithSchema`? Same question for filter.
##########
src/iceberg/manifest_reader.h:
##########
@@ -43,10 +44,27 @@ class ICEBERG_EXPORT ManifestReader {
/// \brief Read manifest files from a manifest list file.
class ICEBERG_EXPORT ManifestListReader {
public:
+ virtual ~ManifestListReader() = default;
virtual Result<std::span<std::unique_ptr<ManifestFile>>> Files() const = 0;
private:
std::unique_ptr<StructLikeReader> reader_;
};
+/// \brief Creates a reader for the manifest list.
+/// \param file_path Path to the manifest list file.
+/// \return A Result containing the reader or an error.
+Result<std::unique_ptr<ManifestListReader>> CreateManifestListReader(
+ const std::string& file_path) {
Review Comment:
BTW, should we use `std::string_view` for path?
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+class ICEBERG_EXPORT ScanTask {
+ public:
+ virtual ~ScanTask() = default;
+
+ /// \brief The number of bytes that should be read by this scan task.
+ virtual int64_t size_bytes() const = 0;
+
+ /// \brief The number of files that should be read by this scan task.
+ virtual int32_t files_count() const = 0;
+
+ /// \brief The number of rows that should be read by this scan task.
+ virtual int64_t estimated_row_count() const = 0;
+};
+
+/// \brief Represents a task to scan a portion of a data file.
+class ICEBERG_EXPORT FileScanTask : public ScanTask {
+ public:
+ FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files, int64_t start,
+ int64_t length, std::shared_ptr<Expression> residual);
+
+ /// \brief The data file that should be read by this scan task.
+ const std::shared_ptr<DataFile>& data_file() const;
+
+ /// \brief The delete files that should be read by this scan task.
+ const std::vector<std::shared_ptr<DataFile>>& delete_files() const;
+
+ /// \brief The byte offset in the data file where the scan should start.
+ int64_t start() const;
+
+ /// \brief The length in bytes to scan from the start offset.
+ int64_t length() const;
+
+ /// \brief The residual expression to apply after scanning the data file.
+ const std::shared_ptr<Expression>& residual() const;
+
+ int64_t size_bytes() const override;
+ int32_t files_count() const override;
+ int64_t estimated_row_count() const override;
+
+ private:
+ std::shared_ptr<DataFile> data_file_; ///< Data file metadata.
+ std::vector<std::shared_ptr<DataFile>> delete_files_; ///< Delete files metadata.
+
+ int64_t start_; ///< Start byte offset.
+ int64_t length_; ///< Length in bytes to scan.
+
+ std::shared_ptr<Expression> residual_; ///< Residual expression to apply.
+};
+
+/// \brief Scan context holding snapshot and scan-specific metadata.
+struct TableScanContext {
+ std::shared_ptr<TableMetadata> table_metadata; ///< Table metadata.
+ std::shared_ptr<Snapshot> snapshot; ///< Snapshot to scan.
+ std::shared_ptr<Schema> projected_schema; ///< Projected schema.
+ std::shared_ptr<Expression> filter; ///< Filter expression to apply.
+ bool case_sensitive = false; ///< Whether the scan is case-sensitive.
+ std::unordered_map<std::string, std::string>
+ options; ///< Additional options for the scan.
+ std::optional<int64_t> limit; ///< Optional limit on the number of rows to scan.
+};
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table The table to scan.
+ /// \param table_metadata The metadata of the table to scan.
+ explicit TableScanBuilder(const Table& table,
+ std::shared_ptr<TableMetadata> table_metadata);
+
+ /// \brief Sets the snapshot ID to scan.
+ /// \param snapshot_id The ID of the snapshot.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSnapshotId(int64_t snapshot_id);
+
+ /// \brief Selects columns to include in the scan.
+ /// \param column_names A list of column names. If empty, all columns will be selected.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithColumnNames(std::vector<std::string> column_names);
+
+ /// \brief Sets the schema to use for the scan.
+ /// \param schema The schema to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSchema(std::shared_ptr<Schema> schema);
+
+ /// \brief Applies a filter expression to the scan.
+ /// \param filter Filter expression to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithFilter(std::shared_ptr<Expression> filter);
+
+ /// \brief Sets whether the scan should be case-sensitive.
+ /// \param case_sensitive Whether the scan is case-sensitive.
+ /// /return Reference to the builder.
+ TableScanBuilder& WithCaseSensitive(bool case_sensitive);
+
+ /// \brief Sets an option for the scan.
+ /// \param property The name of the option.
+ /// \param value The value of the option.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithOption(std::string property, std::string value);
+
+ /// \brief Sets an optional limit on the number of rows to scan.
+ /// \param limit Optional limit on the number of rows.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithLimit(std::optional<int64_t> limit);
+
+ /// \brief Builds and returns a TableScan instance.
+ /// \return A Result containing the TableScan or an error.
+ Result<std::unique_ptr<TableScan>> Build();
+
+ private:
+ const Table& table_; ///< Reference to the table to scan.
+ std::vector<std::string> column_names_;
+ std::optional<int64_t> snapshot_id_;
+ TableScanContext context_; ///< Context for the scan.
+};
+
+/// \brief Represents a configured scan operation on a table.
+class ICEBERG_EXPORT TableScan {
+ public:
+ virtual ~TableScan() = default;
+
+ /// \brief Constructs a TableScan with the given context and file I/O.
+ /// \param context Scan context including snapshot, schema, and filter.
+ /// \param file_io File I/O instance for reading manifests and data files.
+ TableScan(TableScanContext context, std::shared_ptr<FileIO> file_io);
+
+ /// \brief Returns the snapshot being scanned.
+ /// \return A shared pointer to the snapshot.
+ const std::shared_ptr<Snapshot>& snapshot() const;
+
+ /// \brief Returns the projected schema for the scan.
+ /// \return A shared pointer to the projected schema.
+ const std::shared_ptr<Schema>& projection() const;
+
+ /// \brief Returns the scan context.
+ /// \return A reference to the TableScanContext.
+ const TableScanContext& context() const;
+
+ /// \brief Returns the file I/O instance used for reading manifests and data files.
+ /// \return A shared pointer to the FileIO instance.
+ const std::shared_ptr<FileIO>& io() const;
+
+ /// \brief Plans the scan tasks by resolving manifests and data files.
+ /// \return A Result containing scan tasks or an error.
+ virtual Result<std::vector<std::shared_ptr<FileScanTask>>> PlanFiles() const = 0;
+
+ protected:
+ const TableScanContext context_;
+ std::shared_ptr<FileIO> file_io_;
+};
+
+class ICEBERG_EXPORT DataScan : public TableScan {
+ public:
+ DataScan(TableScanContext context, std::shared_ptr<FileIO> file_io);
+
+ /// \brief Plans the scan tasks by resolving manifests and data files.
+ /// \return A Result containing scan tasks or an error.
+ Result<std::vector<std::shared_ptr<FileScanTask>>> PlanFiles() const override;
+
+ private:
+ // Use indexed data structures for efficient lookups
+ struct DeleteFileIndex {
Review Comment:
Should we just move them to source file since they are private and static?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "gty404 (via GitHub)" <gi...@apache.org>.
gty404 commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2113019823
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/expression/expression.h"
+#include "iceberg/file_io.h"
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table Reference to the table to scan.
+ explicit TableScanBuilder(const Table& table);
+
+ /// \brief Sets the snapshot ID to scan.
+ /// \param snapshot_id The ID of the snapshot.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSnapshotId(int64_t snapshot_id);
+
+ /// \brief Selects columns to include in the scan.
+ /// \param column_names A list of column names.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithColumnNames(const std::vector<std::string>& column_names);
+
+ /// \brief Applies a filter expression to the scan.
+ /// \param filter Filter expression to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithFilter(const std::shared_ptr<Expression>& filter);
+
+ /// \brief Builds and returns a TableScan instance.
+ /// \return A Result containing the TableScan or an error.
+ Result<std::unique_ptr<TableScan>> Build();
+
+ private:
+ const Table& table_;
+ std::vector<std::string> column_names_;
+ std::optional<int64_t> snapshot_id_;
+ std::shared_ptr<Expression> filter_;
+};
+
+/// \brief Represents a configured scan operation on a table.
+class ICEBERG_EXPORT TableScan {
+ public:
+ /// \brief Scan context holding snapshot and scan-specific metadata.
+ struct ScanContext {
+ std::shared_ptr<Snapshot> snapshot_; ///< Snapshot to scan.
+ std::shared_ptr<Schema> schema_; ///< Projected schema.
+ std::vector<int32_t> field_ids_; ///< Field IDs of selected columns.
+ std::shared_ptr<Expression> filter_; ///< Filter expression to apply.
+ };
+
+ /// \brief Constructs a TableScan with the given context and file I/O.
+ /// \param context Scan context including snapshot, schema, and filter.
+ /// \param file_io File I/O instance for reading manifests and data files.
+ TableScan(std::unique_ptr<ScanContext> context, std::shared_ptr<FileIO> file_io);
Review Comment:
There is no special reason, I will changed to using struct
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+TableScanBuilder::TableScanBuilder(const Table& table) : table_(table) {}
+
+TableScanBuilder& TableScanBuilder::WithColumnNames(
+ const std::vector<std::string>& column_names) {
+ column_names_ = column_names;
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithSnapshotId(int64_t snapshot_id) {
+ snapshot_id_ = snapshot_id;
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithFilter(
+ const std::shared_ptr<Expression>& filter) {
+ filter_ = filter;
+ return *this;
+}
+
+Result<std::unique_ptr<TableScan>> TableScanBuilder::Build() {
+ ICEBERG_ASSIGN_OR_RAISE(auto snapshot, snapshot_id_ ? table_.snapshot(*snapshot_id_)
+ : Result<std::shared_ptr<Snapshot>>(
+ table_.current_snapshot()));
+
+ auto ResolveSchema = [&]() -> Result<std::shared_ptr<Schema>> {
+ if (snapshot->schema_id) {
+ const auto& schemas = table_.schemas();
+ if (auto it = schemas.find(*snapshot->schema_id); it != schemas.end()) {
+ return it->second;
+ }
+ return InvalidData("Schema {} in snapshot {} is not found", *snapshot->schema_id,
+ snapshot->snapshot_id);
+ }
+ return table_.schema();
+ };
+
+ ICEBERG_ASSIGN_OR_RAISE(auto schema, ResolveSchema());
Review Comment:
I will modify here, thanks.
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/expression/expression.h"
+#include "iceberg/file_io.h"
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table Reference to the table to scan.
+ explicit TableScanBuilder(const Table& table);
+
+ /// \brief Sets the snapshot ID to scan.
+ /// \param snapshot_id The ID of the snapshot.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSnapshotId(int64_t snapshot_id);
+
+ /// \brief Selects columns to include in the scan.
+ /// \param column_names A list of column names.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithColumnNames(const std::vector<std::string>& column_names);
+
+ /// \brief Applies a filter expression to the scan.
+ /// \param filter Filter expression to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithFilter(const std::shared_ptr<Expression>& filter);
+
+ /// \brief Builds and returns a TableScan instance.
+ /// \return A Result containing the TableScan or an error.
+ Result<std::unique_ptr<TableScan>> Build();
+
+ private:
+ const Table& table_;
+ std::vector<std::string> column_names_;
+ std::optional<int64_t> snapshot_id_;
+ std::shared_ptr<Expression> filter_;
+};
+
+/// \brief Represents a configured scan operation on a table.
+class ICEBERG_EXPORT TableScan {
+ public:
+ /// \brief Scan context holding snapshot and scan-specific metadata.
+ struct ScanContext {
+ std::shared_ptr<Snapshot> snapshot_; ///< Snapshot to scan.
+ std::shared_ptr<Schema> schema_; ///< Projected schema.
+ std::vector<int32_t> field_ids_; ///< Field IDs of selected columns.
+ std::shared_ptr<Expression> filter_; ///< Filter expression to apply.
+ };
+
+ /// \brief Constructs a TableScan with the given context and file I/O.
+ /// \param context Scan context including snapshot, schema, and filter.
+ /// \param file_io File I/O instance for reading manifests and data files.
+ TableScan(std::unique_ptr<ScanContext> context, std::shared_ptr<FileIO> file_io);
+
+ /// \brief Plans the scan tasks by resolving manifests and data files.
+ ///
+ /// Returns a list of file scan tasks if successful.
+ /// \return A Result containing scan tasks or an error.
+ Result<std::vector<std::shared_ptr<FileScanTask>>> PlanFiles() const;
+
+ private:
+ /// \brief Creates a reader for the manifest list.
+ /// \param file_path Path to the manifest list file.
+ /// \return A Result containing the reader or an error.
+ Result<std::unique_ptr<ManifestListReader>> CreateManifestListReader(
+ const std::string& file_path) const;
+
+ /// \brief Creates a reader for a manifest file.
+ /// \param file_path Path to the manifest file.
+ /// \return A Result containing the reader or an error.
+ Result<std::unique_ptr<ManifestReader>> CreateManifestReader(
+ const std::string& file_path) const;
+
+ std::unique_ptr<ScanContext> context_;
+ std::shared_ptr<FileIO> file_io_;
+};
+
+/// \brief Represents a task to scan a portion of a data file.
+struct ICEBERG_EXPORT FileScanTask {
+ std::string file_path; ///< Path to the data file.
+ uint64_t start; ///< Start byte offset.
+ uint64_t length; ///< Length in bytes to scan.
+ std::optional<uint64_t> record_count; ///< Optional number of records.
+ DataFile::Content file_content; ///< Type of file content.
+ FileFormatType file_format; ///< Format of the data file.
+ std::shared_ptr<Schema> schema; ///< Projected schema.
+ std::vector<int32_t> field_ids; ///< Field IDs to project.
Review Comment:
The schema includes all columns, and field_ids includes the projected columns.
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/expression/expression.h"
+#include "iceberg/file_io.h"
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
Review Comment:
Yes, in the Java implementation, the hierarchy of Scan and ScanTask has been defined. I will review this part and try to make it more abstract.
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/expression/expression.h"
+#include "iceberg/file_io.h"
+#include "iceberg/manifest_entry.h"
Review Comment:
DataFileContent was modified to be a nested type in [#110](https://github.com/apache/iceberg-cpp/pull/110/) Will there be other issues if it is reverted back?
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/expression/expression.h"
+#include "iceberg/file_io.h"
Review Comment:
The forward declaration of Expression did not appear in type_fwd.h, I will add it later and then delete it here.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "gty404 (via GitHub)" <gi...@apache.org>.
gty404 commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2177839683
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+/// \brief Represents a task to scan a table or a portion of it.
+class ICEBERG_EXPORT ScanTask {
+ public:
+ virtual ~ScanTask() = default;
+
+ /// \brief The number of bytes that should be read by this scan task.
+ virtual int64_t size_bytes() const = 0;
+
+ /// \brief The number of files that should be read by this scan task.
+ virtual int32_t files_count() const = 0;
+
+ /// \brief The number of rows that should be read by this scan task.
+ virtual int64_t estimated_row_count() const = 0;
+};
+
+/// \brief Represents a task to scan a portion of a data file.
+class ICEBERG_EXPORT FileScanTask : public ScanTask {
Review Comment:
1. I initially expected it to just be a struct, but since the previous comments suggested doing an abstraction, I referred to the design in iceberg-java/iceberg-python.
3. Partition spec and value can be obtained from DataFile and Snapshot, and we can add these interfaces when needed for subsequent PR
4. Sure, I will modify it to optional, thanks.
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+namespace {
+/// \brief Use indexed data structures for efficient lookups
+struct DeleteFileIndex {
+ /// \brief Index by sequence number for quick filtering
+ std::multimap<int64_t, ManifestEntry*> sequence_index;
+
+ /// \brief Build the index from a list of manifest entries.
+ void BuildIndex(const std::vector<std::unique_ptr<ManifestEntry>>& entries) {
+ sequence_index.clear();
+
+ for (const auto& entry : entries) {
+ const int64_t seq_num =
+ entry->sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ sequence_index.emplace(seq_num, entry.get());
+ }
+ }
+
+ /// \brief Find delete files that match the sequence number of a data entry.
+ std::vector<ManifestEntry*> FindRelevantEntries(const ManifestEntry& data_entry) const {
+ std::vector<ManifestEntry*> relevant_deletes;
+
+ // Use lower_bound for efficient range search
+ auto data_sequence_number =
+ data_entry.sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ for (auto it = sequence_index.lower_bound(data_sequence_number);
+ it != sequence_index.end(); ++it) {
+ // Additional filtering logic here
Review Comment:
DataFiles only need to retain DeleteFiles with a sequence greater than their own?
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+/// \brief Represents a task to scan a table or a portion of it.
+class ICEBERG_EXPORT ScanTask {
+ public:
+ virtual ~ScanTask() = default;
+
+ /// \brief The number of bytes that should be read by this scan task.
+ virtual int64_t size_bytes() const = 0;
+
+ /// \brief The number of files that should be read by this scan task.
+ virtual int32_t files_count() const = 0;
+
+ /// \brief The number of rows that should be read by this scan task.
+ virtual int64_t estimated_row_count() const = 0;
+};
+
+/// \brief Represents a task to scan a portion of a data file.
+class ICEBERG_EXPORT FileScanTask : public ScanTask {
+ public:
+ FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files, int64_t start,
+ int64_t length, std::shared_ptr<Expression> residual);
+
+ /// \brief The data file that should be read by this scan task.
+ const std::shared_ptr<DataFile>& data_file() const;
+
+ /// \brief The delete files that should be read by this scan task.
+ const std::vector<std::shared_ptr<DataFile>>& delete_files() const;
+
+ /// \brief The byte offset in the data file where the scan should start.
+ int64_t start() const;
+
+ /// \brief The length in bytes to scan from the start offset.
+ int64_t length() const;
+
+ /// \brief The residual expression to apply after scanning the data file.
+ const std::shared_ptr<Expression>& residual() const;
+
+ int64_t size_bytes() const override;
+ int32_t files_count() const override;
+ int64_t estimated_row_count() const override;
+
+ private:
+ /// \brief Data file metadata.
+ std::shared_ptr<DataFile> data_file_;
+ /// \brief Delete files metadata.
+ std::vector<std::shared_ptr<DataFile>> delete_files_;
+ /// \brief Start byte offset.
+ int64_t start_;
+ /// \brief Length in bytes to scan.
+ int64_t length_;
+ /// \brief Residual expression to apply.
+ std::shared_ptr<Expression> residual_;
+};
+
+/// \brief Scan context holding snapshot and scan-specific metadata.
+struct TableScanContext {
+ /// \brief Table metadata.
+ std::shared_ptr<TableMetadata> table_metadata;
+ /// \brief Snapshot to scan.
+ std::shared_ptr<Snapshot> snapshot;
+ /// \brief Projected schema.
+ std::shared_ptr<Schema> projected_schema;
+ /// \brief Filter expression to apply.
+ std::shared_ptr<Expression> filter;
+ /// \brief Whether the scan is case-sensitive.
+ bool case_sensitive = false;
+ /// \brief Additional options for the scan.
+ std::unordered_map<std::string, std::string> options;
+ /// \brief Optional limit on the number of rows to scan.
+ std::optional<int64_t> limit;
+};
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table The table to scan.
+ /// \param table_metadata The metadata of the table to scan.
+ explicit TableScanBuilder(const Table& table,
+ std::shared_ptr<TableMetadata> table_metadata);
+
+ /// \brief Sets the snapshot ID to scan.
+ /// \param snapshot_id The ID of the snapshot.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSnapshotId(int64_t snapshot_id);
+
+ /// \brief Selects columns to include in the scan.
+ /// \param column_names A list of column names. If empty, all columns will be selected.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithColumnNames(std::vector<std::string> column_names);
+
+ /// \brief Sets the schema to use for the scan.
+ /// \param schema The schema to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSchema(std::shared_ptr<Schema> schema);
+
+ /// \brief Applies a filter expression to the scan.
+ /// \param filter Filter expression to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithFilter(std::shared_ptr<Expression> filter);
+
+ /// \brief Sets whether the scan should be case-sensitive.
+ /// \param case_sensitive Whether the scan is case-sensitive.
+ /// /return Reference to the builder.
+ TableScanBuilder& WithCaseSensitive(bool case_sensitive);
+
+ /// \brief Sets an option for the scan.
+ /// \param property The name of the option.
+ /// \param value The value of the option.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithOption(std::string property, std::string value);
+
+ /// \brief Sets an optional limit on the number of rows to scan.
+ /// \param limit Optional limit on the number of rows.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithLimit(std::optional<int64_t> limit);
+
+ /// \brief Builds and returns a TableScan instance.
+ /// \return A Result containing the TableScan or an error.
+ Result<std::unique_ptr<TableScan>> Build();
+
+ private:
+ /// \brief Reference to the table to scan.
+ const Table& table_;
+ /// \brief column names to project in the scan.
+ std::vector<std::string> column_names_;
+ /// \brief snapshot ID to scan, if specified.
+ std::optional<int64_t> snapshot_id_;
+ /// \brief Context for the scan, including snapshot, schema, and filter.
+ TableScanContext context_;
Review Comment:
I originally expected that TableScanContext would be context information retained after converting various input parameters, and that what was no longer needed in the subsequent file scanning process would be removed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "mapleFU (via GitHub)" <gi...@apache.org>.
mapleFU commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2185357548
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,285 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+namespace {
+/// \brief Use indexed data structures for efficient lookups
+class DeleteFileIndex {
+ public:
+ /// \brief Build the index from a list of manifest entries.
+ explicit DeleteFileIndex(const std::vector<std::unique_ptr<ManifestEntry>>& entries) {
+ for (const auto& entry : entries) {
+ const int64_t seq_num =
+ entry->sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ sequence_index.emplace(seq_num, entry.get());
+ }
+ }
+
+ /// \brief Find delete files that match the sequence number of a data entry.
+ std::vector<ManifestEntry*> FindRelevantEntries(const ManifestEntry& data_entry) const {
+ std::vector<ManifestEntry*> relevant_deletes;
+
+ // Use lower_bound for efficient range search
+ auto data_sequence_number =
+ data_entry.sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ for (auto it = sequence_index.lower_bound(data_sequence_number);
+ it != sequence_index.end(); ++it) {
+ // Additional filtering logic here
+ relevant_deletes.push_back(it->second);
+ }
+
+ return relevant_deletes;
+ }
+
+ private:
+ /// \brief Index by sequence number for quick filtering
+ std::multimap<int64_t, ManifestEntry*> sequence_index;
+};
+
+/// \brief Get matched delete files for a given data entry.
+std::vector<std::shared_ptr<DataFile>> GetMatchedDeletes(
+ const ManifestEntry& data_entry, const DeleteFileIndex& delete_file_index) {
+ const auto relevant_entries = delete_file_index.FindRelevantEntries(data_entry);
+ std::vector<std::shared_ptr<DataFile>> matched_deletes;
+ if (relevant_entries.empty()) {
+ return matched_deletes;
+ }
+
+ matched_deletes.reserve(relevant_entries.size());
+ for (const auto& delete_entry : relevant_entries) {
+ // TODO(gty404): check if the delete entry contains the data entry's file path
+ matched_deletes.emplace_back(delete_entry->data_file);
+ }
+ return matched_deletes;
+}
+} // namespace
+
+// implement FileScanTask
+FileScanTask::FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files,
+ std::shared_ptr<Expression> residual)
+ : data_file_(std::move(file)),
+ delete_files_(std::move(delete_files)),
+ residual_(std::move(residual)) {}
+
+const std::shared_ptr<DataFile>& FileScanTask::data_file() const { return data_file_; }
+
+const std::vector<std::shared_ptr<DataFile>>& FileScanTask::delete_files() const {
+ return delete_files_;
+}
+
+int64_t FileScanTask::SizeBytes() const {
+ int64_t sizeInBytes = data_file_->file_size_in_bytes;
Review Comment:
```suggestion
int64_t size_in_bytes = data_file_->file_size_in_bytes;
```
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,285 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+namespace {
+/// \brief Use indexed data structures for efficient lookups
+class DeleteFileIndex {
+ public:
+ /// \brief Build the index from a list of manifest entries.
+ explicit DeleteFileIndex(const std::vector<std::unique_ptr<ManifestEntry>>& entries) {
+ for (const auto& entry : entries) {
+ const int64_t seq_num =
+ entry->sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ sequence_index.emplace(seq_num, entry.get());
+ }
+ }
+
+ /// \brief Find delete files that match the sequence number of a data entry.
+ std::vector<ManifestEntry*> FindRelevantEntries(const ManifestEntry& data_entry) const {
+ std::vector<ManifestEntry*> relevant_deletes;
+
+ // Use lower_bound for efficient range search
+ auto data_sequence_number =
+ data_entry.sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ for (auto it = sequence_index.lower_bound(data_sequence_number);
+ it != sequence_index.end(); ++it) {
+ // Additional filtering logic here
+ relevant_deletes.push_back(it->second);
+ }
+
+ return relevant_deletes;
+ }
+
+ private:
+ /// \brief Index by sequence number for quick filtering
+ std::multimap<int64_t, ManifestEntry*> sequence_index;
+};
+
+/// \brief Get matched delete files for a given data entry.
+std::vector<std::shared_ptr<DataFile>> GetMatchedDeletes(
+ const ManifestEntry& data_entry, const DeleteFileIndex& delete_file_index) {
+ const auto relevant_entries = delete_file_index.FindRelevantEntries(data_entry);
+ std::vector<std::shared_ptr<DataFile>> matched_deletes;
+ if (relevant_entries.empty()) {
+ return matched_deletes;
+ }
+
+ matched_deletes.reserve(relevant_entries.size());
+ for (const auto& delete_entry : relevant_entries) {
+ // TODO(gty404): check if the delete entry contains the data entry's file path
+ matched_deletes.emplace_back(delete_entry->data_file);
+ }
+ return matched_deletes;
+}
+} // namespace
+
+// implement FileScanTask
+FileScanTask::FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files,
+ std::shared_ptr<Expression> residual)
+ : data_file_(std::move(file)),
+ delete_files_(std::move(delete_files)),
+ residual_(std::move(residual)) {}
+
+const std::shared_ptr<DataFile>& FileScanTask::data_file() const { return data_file_; }
+
+const std::vector<std::shared_ptr<DataFile>>& FileScanTask::delete_files() const {
+ return delete_files_;
+}
+
+int64_t FileScanTask::SizeBytes() const {
+ int64_t sizeInBytes = data_file_->file_size_in_bytes;
+ std::ranges::for_each(delete_files_, [&sizeInBytes](const auto& delete_file) {
+ sizeInBytes += delete_file->file_size_in_bytes;
+ });
+ return sizeInBytes;
+}
+
+int32_t FileScanTask::FilesCount() const {
+ return static_cast<int32_t>(delete_files_.size() + 1);
+}
+
+int64_t FileScanTask::EstimatedRowCount() const {
+ if (data_file_->file_size_in_bytes == 0) {
+ return 0;
+ }
+ const auto sizeInBytes = data_file_->file_size_in_bytes;
+ const double scannedFileFraction =
+ static_cast<double>(sizeInBytes) / data_file_->file_size_in_bytes;
+ return static_cast<int64_t>(scannedFileFraction * data_file_->record_count);
+}
+
+const std::shared_ptr<Expression>& FileScanTask::residual() const { return residual_; }
+
+TableScanBuilder::TableScanBuilder(std::shared_ptr<TableMetadata> table_metadata,
+ std::shared_ptr<FileIO> file_io)
+ : file_io_(std::move(file_io)) {
+ context_.table_metadata = std::move(table_metadata);
+}
+
+TableScanBuilder& TableScanBuilder::WithColumnNames(
+ std::vector<std::string> column_names) {
+ column_names_ = std::move(column_names);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithProjectedSchema(std::shared_ptr<Schema> schema) {
+ context_.projected_schema = std::move(schema);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithSnapshotId(int64_t snapshot_id) {
+ snapshot_id_ = snapshot_id;
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithFilter(std::shared_ptr<Expression> filter) {
+ context_.filter = std::move(filter);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithCaseSensitive(bool case_sensitive) {
+ context_.case_sensitive = case_sensitive;
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithOption(std::string property, std::string value) {
+ context_.options[std::move(property)] = std::move(value);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithLimit(std::optional<int64_t> limit) {
+ context_.limit = limit;
+ return *this;
+}
+
+Result<std::unique_ptr<TableScan>> TableScanBuilder::Build() {
+ const auto& table_metadata = context_.table_metadata;
+ auto snapshot_id = snapshot_id_ ? snapshot_id_ : table_metadata->current_snapshot_id;
+ if (!snapshot_id) {
+ return InvalidArgument("No snapshot ID specified for table {}",
+ table_metadata->table_uuid);
+ }
+ auto iter = std::ranges::find_if(table_metadata->snapshots,
+ [&snapshot_id](const auto& snapshot) {
Review Comment:
Use the way below to eval `*snapshot_id` only once
```suggestion
[id = *snapshot_id](const auto& snapshot) {
```
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,285 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+namespace {
+/// \brief Use indexed data structures for efficient lookups
+class DeleteFileIndex {
Review Comment:
In intuition this can build once ( push all element and sort one), then when querying, return a slice for it. But this also looks good to me
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,285 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+namespace {
+/// \brief Use indexed data structures for efficient lookups
+class DeleteFileIndex {
+ public:
+ /// \brief Build the index from a list of manifest entries.
+ explicit DeleteFileIndex(const std::vector<std::unique_ptr<ManifestEntry>>& entries) {
+ for (const auto& entry : entries) {
+ const int64_t seq_num =
+ entry->sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ sequence_index.emplace(seq_num, entry.get());
+ }
+ }
+
+ /// \brief Find delete files that match the sequence number of a data entry.
+ std::vector<ManifestEntry*> FindRelevantEntries(const ManifestEntry& data_entry) const {
+ std::vector<ManifestEntry*> relevant_deletes;
+
+ // Use lower_bound for efficient range search
+ auto data_sequence_number =
+ data_entry.sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ for (auto it = sequence_index.lower_bound(data_sequence_number);
+ it != sequence_index.end(); ++it) {
+ // Additional filtering logic here
+ relevant_deletes.push_back(it->second);
+ }
+
+ return relevant_deletes;
+ }
+
+ private:
+ /// \brief Index by sequence number for quick filtering
+ std::multimap<int64_t, ManifestEntry*> sequence_index;
+};
+
+/// \brief Get matched delete files for a given data entry.
+std::vector<std::shared_ptr<DataFile>> GetMatchedDeletes(
+ const ManifestEntry& data_entry, const DeleteFileIndex& delete_file_index) {
+ const auto relevant_entries = delete_file_index.FindRelevantEntries(data_entry);
+ std::vector<std::shared_ptr<DataFile>> matched_deletes;
+ if (relevant_entries.empty()) {
+ return matched_deletes;
+ }
+
+ matched_deletes.reserve(relevant_entries.size());
+ for (const auto& delete_entry : relevant_entries) {
+ // TODO(gty404): check if the delete entry contains the data entry's file path
+ matched_deletes.emplace_back(delete_entry->data_file);
+ }
+ return matched_deletes;
+}
+} // namespace
+
+// implement FileScanTask
+FileScanTask::FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files,
+ std::shared_ptr<Expression> residual)
+ : data_file_(std::move(file)),
+ delete_files_(std::move(delete_files)),
+ residual_(std::move(residual)) {}
+
+const std::shared_ptr<DataFile>& FileScanTask::data_file() const { return data_file_; }
+
+const std::vector<std::shared_ptr<DataFile>>& FileScanTask::delete_files() const {
+ return delete_files_;
+}
+
+int64_t FileScanTask::SizeBytes() const {
+ int64_t sizeInBytes = data_file_->file_size_in_bytes;
+ std::ranges::for_each(delete_files_, [&sizeInBytes](const auto& delete_file) {
+ sizeInBytes += delete_file->file_size_in_bytes;
+ });
+ return sizeInBytes;
+}
+
+int32_t FileScanTask::FilesCount() const {
+ return static_cast<int32_t>(delete_files_.size() + 1);
+}
+
+int64_t FileScanTask::EstimatedRowCount() const {
+ if (data_file_->file_size_in_bytes == 0) {
+ return 0;
+ }
+ const auto sizeInBytes = data_file_->file_size_in_bytes;
+ const double scannedFileFraction =
+ static_cast<double>(sizeInBytes) / data_file_->file_size_in_bytes;
+ return static_cast<int64_t>(scannedFileFraction * data_file_->record_count);
+}
+
+const std::shared_ptr<Expression>& FileScanTask::residual() const { return residual_; }
+
+TableScanBuilder::TableScanBuilder(std::shared_ptr<TableMetadata> table_metadata,
+ std::shared_ptr<FileIO> file_io)
+ : file_io_(std::move(file_io)) {
+ context_.table_metadata = std::move(table_metadata);
+}
+
+TableScanBuilder& TableScanBuilder::WithColumnNames(
+ std::vector<std::string> column_names) {
+ column_names_ = std::move(column_names);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithProjectedSchema(std::shared_ptr<Schema> schema) {
+ context_.projected_schema = std::move(schema);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithSnapshotId(int64_t snapshot_id) {
+ snapshot_id_ = snapshot_id;
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithFilter(std::shared_ptr<Expression> filter) {
+ context_.filter = std::move(filter);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithCaseSensitive(bool case_sensitive) {
+ context_.case_sensitive = case_sensitive;
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithOption(std::string property, std::string value) {
+ context_.options[std::move(property)] = std::move(value);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithLimit(std::optional<int64_t> limit) {
+ context_.limit = limit;
+ return *this;
+}
+
+Result<std::unique_ptr<TableScan>> TableScanBuilder::Build() {
+ const auto& table_metadata = context_.table_metadata;
+ auto snapshot_id = snapshot_id_ ? snapshot_id_ : table_metadata->current_snapshot_id;
+ if (!snapshot_id) {
+ return InvalidArgument("No snapshot ID specified for table {}",
+ table_metadata->table_uuid);
+ }
+ auto iter = std::ranges::find_if(table_metadata->snapshots,
+ [&snapshot_id](const auto& snapshot) {
+ return snapshot->snapshot_id == *snapshot_id;
+ });
+ if (iter == table_metadata->snapshots.end() || *iter == nullptr) {
+ return NotFound("Snapshot with ID {} is not found", *snapshot_id);
+ }
+ context_.snapshot = *iter;
+
+ if (!context_.projected_schema) {
+ const auto& snapshot = context_.snapshot;
+ auto schema_id =
+ snapshot->schema_id ? snapshot->schema_id : table_metadata->current_schema_id;
+ if (!schema_id) {
+ return InvalidArgument("No schema ID found in snapshot {} for table {}",
+ snapshot->snapshot_id, table_metadata->table_uuid);
+ }
+
+ const auto& schemas = table_metadata->schemas;
+ const auto it = std::ranges::find_if(schemas, [&schema_id](const auto& schema) {
Review Comment:
ditto
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,285 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+namespace {
+/// \brief Use indexed data structures for efficient lookups
+class DeleteFileIndex {
+ public:
+ /// \brief Build the index from a list of manifest entries.
+ explicit DeleteFileIndex(const std::vector<std::unique_ptr<ManifestEntry>>& entries) {
+ for (const auto& entry : entries) {
+ const int64_t seq_num =
+ entry->sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ sequence_index.emplace(seq_num, entry.get());
+ }
+ }
+
+ /// \brief Find delete files that match the sequence number of a data entry.
+ std::vector<ManifestEntry*> FindRelevantEntries(const ManifestEntry& data_entry) const {
+ std::vector<ManifestEntry*> relevant_deletes;
+
+ // Use lower_bound for efficient range search
+ auto data_sequence_number =
+ data_entry.sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ for (auto it = sequence_index.lower_bound(data_sequence_number);
+ it != sequence_index.end(); ++it) {
+ // Additional filtering logic here
+ relevant_deletes.push_back(it->second);
+ }
+
+ return relevant_deletes;
+ }
+
+ private:
+ /// \brief Index by sequence number for quick filtering
+ std::multimap<int64_t, ManifestEntry*> sequence_index;
+};
+
+/// \brief Get matched delete files for a given data entry.
+std::vector<std::shared_ptr<DataFile>> GetMatchedDeletes(
+ const ManifestEntry& data_entry, const DeleteFileIndex& delete_file_index) {
+ const auto relevant_entries = delete_file_index.FindRelevantEntries(data_entry);
+ std::vector<std::shared_ptr<DataFile>> matched_deletes;
+ if (relevant_entries.empty()) {
+ return matched_deletes;
+ }
+
+ matched_deletes.reserve(relevant_entries.size());
+ for (const auto& delete_entry : relevant_entries) {
+ // TODO(gty404): check if the delete entry contains the data entry's file path
+ matched_deletes.emplace_back(delete_entry->data_file);
+ }
+ return matched_deletes;
+}
+} // namespace
+
+// implement FileScanTask
+FileScanTask::FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files,
+ std::shared_ptr<Expression> residual)
+ : data_file_(std::move(file)),
+ delete_files_(std::move(delete_files)),
+ residual_(std::move(residual)) {}
+
+const std::shared_ptr<DataFile>& FileScanTask::data_file() const { return data_file_; }
+
+const std::vector<std::shared_ptr<DataFile>>& FileScanTask::delete_files() const {
+ return delete_files_;
+}
+
+int64_t FileScanTask::SizeBytes() const {
+ int64_t sizeInBytes = data_file_->file_size_in_bytes;
+ std::ranges::for_each(delete_files_, [&sizeInBytes](const auto& delete_file) {
+ sizeInBytes += delete_file->file_size_in_bytes;
+ });
+ return sizeInBytes;
+}
+
+int32_t FileScanTask::FilesCount() const {
+ return static_cast<int32_t>(delete_files_.size() + 1);
+}
+
+int64_t FileScanTask::EstimatedRowCount() const {
+ if (data_file_->file_size_in_bytes == 0) {
+ return 0;
+ }
+ const auto sizeInBytes = data_file_->file_size_in_bytes;
+ const double scannedFileFraction =
+ static_cast<double>(sizeInBytes) / data_file_->file_size_in_bytes;
+ return static_cast<int64_t>(scannedFileFraction * data_file_->record_count);
+}
+
+const std::shared_ptr<Expression>& FileScanTask::residual() const { return residual_; }
+
+TableScanBuilder::TableScanBuilder(std::shared_ptr<TableMetadata> table_metadata,
+ std::shared_ptr<FileIO> file_io)
+ : file_io_(std::move(file_io)) {
+ context_.table_metadata = std::move(table_metadata);
+}
+
+TableScanBuilder& TableScanBuilder::WithColumnNames(
+ std::vector<std::string> column_names) {
+ column_names_ = std::move(column_names);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithProjectedSchema(std::shared_ptr<Schema> schema) {
+ context_.projected_schema = std::move(schema);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithSnapshotId(int64_t snapshot_id) {
+ snapshot_id_ = snapshot_id;
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithFilter(std::shared_ptr<Expression> filter) {
+ context_.filter = std::move(filter);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithCaseSensitive(bool case_sensitive) {
+ context_.case_sensitive = case_sensitive;
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithOption(std::string property, std::string value) {
+ context_.options[std::move(property)] = std::move(value);
+ return *this;
+}
+
+TableScanBuilder& TableScanBuilder::WithLimit(std::optional<int64_t> limit) {
+ context_.limit = limit;
+ return *this;
+}
+
+Result<std::unique_ptr<TableScan>> TableScanBuilder::Build() {
+ const auto& table_metadata = context_.table_metadata;
+ auto snapshot_id = snapshot_id_ ? snapshot_id_ : table_metadata->current_snapshot_id;
+ if (!snapshot_id) {
+ return InvalidArgument("No snapshot ID specified for table {}",
+ table_metadata->table_uuid);
+ }
+ auto iter = std::ranges::find_if(table_metadata->snapshots,
+ [&snapshot_id](const auto& snapshot) {
+ return snapshot->snapshot_id == *snapshot_id;
+ });
+ if (iter == table_metadata->snapshots.end() || *iter == nullptr) {
+ return NotFound("Snapshot with ID {} is not found", *snapshot_id);
+ }
+ context_.snapshot = *iter;
+
+ if (!context_.projected_schema) {
+ const auto& snapshot = context_.snapshot;
+ auto schema_id =
+ snapshot->schema_id ? snapshot->schema_id : table_metadata->current_schema_id;
+ if (!schema_id) {
+ return InvalidArgument("No schema ID found in snapshot {} for table {}",
+ snapshot->snapshot_id, table_metadata->table_uuid);
+ }
+
+ const auto& schemas = table_metadata->schemas;
+ const auto it = std::ranges::find_if(schemas, [&schema_id](const auto& schema) {
+ return schema->schema_id() == *schema_id;
+ });
+ if (it == schemas.end()) {
+ return InvalidArgument("Schema {} in snapshot {} is not found",
+ *snapshot->schema_id, snapshot->snapshot_id);
+ }
+ const auto& schema = *it;
+
+ if (column_names_.empty()) {
+ context_.projected_schema = schema;
+ } else {
+ // TODO(gty404): collect touched columns from filter expression
+ std::vector<SchemaField> projected_fields;
+ projected_fields.reserve(column_names_.size());
+ for (const auto& column_name : column_names_) {
+ // TODO(gty404): support case-insensitive column names
+ auto field_opt = schema->GetFieldByName(column_name);
+ if (!field_opt) {
+ return InvalidArgument("Column {} not found in schema", column_name);
Review Comment:
should we add schema to output?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "zhjwpku (via GitHub)" <gi...@apache.org>.
zhjwpku commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2173250464
##########
src/iceberg/snapshot.h:
##########
@@ -222,6 +222,7 @@ struct ICEBERG_EXPORT DataOperation {
/// Snapshots are created by table operations.
struct ICEBERG_EXPORT Snapshot {
static constexpr int64_t kInvalidSnapshotId = -1;
+ static constexpr int64_t kInitialSequenceNumber = 0;
Review Comment:
There is a TableMetadata::kInitialSequenceNumber, which seems to serve the same purpose here?
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+class ICEBERG_EXPORT ScanTask {
+ public:
+ virtual ~ScanTask() = default;
+
+ /// \brief The number of bytes that should be read by this scan task.
+ virtual int64_t size_bytes() const = 0;
+
+ /// \brief The number of files that should be read by this scan task.
+ virtual int32_t files_count() const = 0;
+
+ /// \brief The number of rows that should be read by this scan task.
+ virtual int64_t estimated_row_count() const = 0;
+};
+
+/// \brief Represents a task to scan a portion of a data file.
+class ICEBERG_EXPORT FileScanTask : public ScanTask {
+ public:
+ FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files, int64_t start,
+ int64_t length, std::shared_ptr<Expression> residual);
+
+ /// \brief The data file that should be read by this scan task.
+ const std::shared_ptr<DataFile>& data_file() const;
+
+ /// \brief The delete files that should be read by this scan task.
+ const std::vector<std::shared_ptr<DataFile>>& delete_files() const;
+
+ /// \brief The byte offset in the data file where the scan should start.
+ int64_t start() const;
+
+ /// \brief The length in bytes to scan from the start offset.
+ int64_t length() const;
+
+ /// \brief The residual expression to apply after scanning the data file.
+ const std::shared_ptr<Expression>& residual() const;
+
+ int64_t size_bytes() const override;
+ int32_t files_count() const override;
+ int64_t estimated_row_count() const override;
+
+ private:
+ std::shared_ptr<DataFile> data_file_; ///< Data file metadata.
Review Comment:
nit: Perhaps keep the comments on a separate line for consistency?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "lishuxu (via GitHub)" <gi...@apache.org>.
lishuxu commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2173350649
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+class ICEBERG_EXPORT ScanTask {
+ public:
+ virtual ~ScanTask() = default;
+
+ /// \brief The number of bytes that should be read by this scan task.
+ virtual int64_t size_bytes() const = 0;
+
+ /// \brief The number of files that should be read by this scan task.
+ virtual int32_t files_count() const = 0;
+
+ /// \brief The number of rows that should be read by this scan task.
+ virtual int64_t estimated_row_count() const = 0;
+};
+
+/// \brief Represents a task to scan a portion of a data file.
+class ICEBERG_EXPORT FileScanTask : public ScanTask {
+ public:
+ FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files, int64_t start,
+ int64_t length, std::shared_ptr<Expression> residual);
+
+ /// \brief The data file that should be read by this scan task.
+ const std::shared_ptr<DataFile>& data_file() const;
+
+ /// \brief The delete files that should be read by this scan task.
+ const std::vector<std::shared_ptr<DataFile>>& delete_files() const;
+
+ /// \brief The byte offset in the data file where the scan should start.
+ int64_t start() const;
+
+ /// \brief The length in bytes to scan from the start offset.
+ int64_t length() const;
+
+ /// \brief The residual expression to apply after scanning the data file.
+ const std::shared_ptr<Expression>& residual() const;
+
+ int64_t size_bytes() const override;
+ int32_t files_count() const override;
+ int64_t estimated_row_count() const override;
+
+ private:
+ std::shared_ptr<DataFile> data_file_; ///< Data file metadata.
+ std::vector<std::shared_ptr<DataFile>> delete_files_; ///< Delete files metadata.
+
+ int64_t start_; ///< Start byte offset.
+ int64_t length_; ///< Length in bytes to scan.
+
+ std::shared_ptr<Expression> residual_; ///< Residual expression to apply.
+};
+
+/// \brief Scan context holding snapshot and scan-specific metadata.
+struct TableScanContext {
+ std::shared_ptr<TableMetadata> table_metadata; ///< Table metadata.
+ std::shared_ptr<Snapshot> snapshot; ///< Snapshot to scan.
+ std::shared_ptr<Schema> projected_schema; ///< Projected schema.
+ std::shared_ptr<Expression> filter; ///< Filter expression to apply.
+ bool case_sensitive = false; ///< Whether the scan is case-sensitive.
+ std::unordered_map<std::string, std::string>
+ options; ///< Additional options for the scan.
+ std::optional<int64_t> limit; ///< Optional limit on the number of rows to scan.
+};
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table The table to scan.
+ /// \param table_metadata The metadata of the table to scan.
+ explicit TableScanBuilder(const Table& table,
+ std::shared_ptr<TableMetadata> table_metadata);
+
+ /// \brief Sets the snapshot ID to scan.
+ /// \param snapshot_id The ID of the snapshot.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSnapshotId(int64_t snapshot_id);
+
+ /// \brief Selects columns to include in the scan.
+ /// \param column_names A list of column names. If empty, all columns will be selected.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithColumnNames(std::vector<std::string> column_names);
+
+ /// \brief Sets the schema to use for the scan.
+ /// \param schema The schema to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSchema(std::shared_ptr<Schema> schema);
+
+ /// \brief Applies a filter expression to the scan.
+ /// \param filter Filter expression to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithFilter(std::shared_ptr<Expression> filter);
+
+ /// \brief Sets whether the scan should be case-sensitive.
+ /// \param case_sensitive Whether the scan is case-sensitive.
+ /// /return Reference to the builder.
+ TableScanBuilder& WithCaseSensitive(bool case_sensitive);
+
+ /// \brief Sets an option for the scan.
+ /// \param property The name of the option.
+ /// \param value The value of the option.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithOption(std::string property, std::string value);
+
+ /// \brief Sets an optional limit on the number of rows to scan.
+ /// \param limit Optional limit on the number of rows.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithLimit(std::optional<int64_t> limit);
+
+ /// \brief Builds and returns a TableScan instance.
+ /// \return A Result containing the TableScan or an error.
+ Result<std::unique_ptr<TableScan>> Build();
+
+ private:
+ const Table& table_; ///< Reference to the table to scan.
+ std::vector<std::string> column_names_;
+ std::optional<int64_t> snapshot_id_;
+ TableScanContext context_; ///< Context for the scan.
+};
+
+/// \brief Represents a configured scan operation on a table.
+class ICEBERG_EXPORT TableScan {
+ public:
+ virtual ~TableScan() = default;
+
+ /// \brief Constructs a TableScan with the given context and file I/O.
+ /// \param context Scan context including snapshot, schema, and filter.
+ /// \param file_io File I/O instance for reading manifests and data files.
+ TableScan(TableScanContext context, std::shared_ptr<FileIO> file_io);
+
+ /// \brief Returns the snapshot being scanned.
+ /// \return A shared pointer to the snapshot.
+ const std::shared_ptr<Snapshot>& snapshot() const;
+
+ /// \brief Returns the projected schema for the scan.
+ /// \return A shared pointer to the projected schema.
+ const std::shared_ptr<Schema>& projection() const;
+
+ /// \brief Returns the scan context.
+ /// \return A reference to the TableScanContext.
+ const TableScanContext& context() const;
+
+ /// \brief Returns the file I/O instance used for reading manifests and data files.
+ /// \return A shared pointer to the FileIO instance.
+ const std::shared_ptr<FileIO>& io() const;
+
+ /// \brief Plans the scan tasks by resolving manifests and data files.
+ /// \return A Result containing scan tasks or an error.
+ virtual Result<std::vector<std::shared_ptr<FileScanTask>>> PlanFiles() const = 0;
Review Comment:
Would it be more appropriate to name this method as PlanTasks?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "gty404 (via GitHub)" <gi...@apache.org>.
gty404 commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2181748161
##########
src/iceberg/table_scan.h:
##########
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+/// \brief Represents a task to scan a table or a portion of it.
+class ICEBERG_EXPORT ScanTask {
+ public:
+ virtual ~ScanTask() = default;
+
+ /// \brief The number of bytes that should be read by this scan task.
+ virtual int64_t size_bytes() const = 0;
+
+ /// \brief The number of files that should be read by this scan task.
+ virtual int32_t files_count() const = 0;
+
+ /// \brief The number of rows that should be read by this scan task.
+ virtual int64_t estimated_row_count() const = 0;
+};
+
+/// \brief Represents a task to scan a portion of a data file.
+class ICEBERG_EXPORT FileScanTask : public ScanTask {
+ public:
+ FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files, int64_t start,
+ int64_t length, std::shared_ptr<Expression> residual);
+
+ /// \brief The data file that should be read by this scan task.
+ const std::shared_ptr<DataFile>& data_file() const;
+
+ /// \brief The delete files that should be read by this scan task.
+ const std::vector<std::shared_ptr<DataFile>>& delete_files() const;
+
+ /// \brief The byte offset in the data file where the scan should start.
+ int64_t start() const;
+
+ /// \brief The length in bytes to scan from the start offset.
+ int64_t length() const;
+
+ /// \brief The residual expression to apply after scanning the data file.
+ const std::shared_ptr<Expression>& residual() const;
+
+ int64_t size_bytes() const override;
+ int32_t files_count() const override;
+ int64_t estimated_row_count() const override;
+
+ private:
+ /// \brief Data file metadata.
+ std::shared_ptr<DataFile> data_file_;
+ /// \brief Delete files metadata.
+ std::vector<std::shared_ptr<DataFile>> delete_files_;
+ /// \brief Start byte offset.
+ int64_t start_;
+ /// \brief Length in bytes to scan.
+ int64_t length_;
+ /// \brief Residual expression to apply.
+ std::shared_ptr<Expression> residual_;
+};
+
+/// \brief Scan context holding snapshot and scan-specific metadata.
+struct TableScanContext {
+ /// \brief Table metadata.
+ std::shared_ptr<TableMetadata> table_metadata;
+ /// \brief Snapshot to scan.
+ std::shared_ptr<Snapshot> snapshot;
+ /// \brief Projected schema.
+ std::shared_ptr<Schema> projected_schema;
+ /// \brief Filter expression to apply.
+ std::shared_ptr<Expression> filter;
+ /// \brief Whether the scan is case-sensitive.
+ bool case_sensitive = false;
+ /// \brief Additional options for the scan.
+ std::unordered_map<std::string, std::string> options;
+ /// \brief Optional limit on the number of rows to scan.
+ std::optional<int64_t> limit;
+};
+
+/// \brief Builder class for creating TableScan instances.
+class ICEBERG_EXPORT TableScanBuilder {
+ public:
+ /// \brief Constructs a TableScanBuilder for the given table.
+ /// \param table The table to scan.
+ /// \param table_metadata The metadata of the table to scan.
+ explicit TableScanBuilder(const Table& table,
+ std::shared_ptr<TableMetadata> table_metadata);
+
+ /// \brief Sets the snapshot ID to scan.
+ /// \param snapshot_id The ID of the snapshot.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSnapshotId(int64_t snapshot_id);
+
+ /// \brief Selects columns to include in the scan.
+ /// \param column_names A list of column names. If empty, all columns will be selected.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithColumnNames(std::vector<std::string> column_names);
+
+ /// \brief Sets the schema to use for the scan.
+ /// \param schema The schema to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithSchema(std::shared_ptr<Schema> schema);
+
+ /// \brief Applies a filter expression to the scan.
+ /// \param filter Filter expression to use.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithFilter(std::shared_ptr<Expression> filter);
+
+ /// \brief Sets whether the scan should be case-sensitive.
+ /// \param case_sensitive Whether the scan is case-sensitive.
+ /// /return Reference to the builder.
+ TableScanBuilder& WithCaseSensitive(bool case_sensitive);
+
+ /// \brief Sets an option for the scan.
+ /// \param property The name of the option.
+ /// \param value The value of the option.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithOption(std::string property, std::string value);
+
+ /// \brief Sets an optional limit on the number of rows to scan.
+ /// \param limit Optional limit on the number of rows.
+ /// \return Reference to the builder.
+ TableScanBuilder& WithLimit(std::optional<int64_t> limit);
+
+ /// \brief Builds and returns a TableScan instance.
+ /// \return A Result containing the TableScan or an error.
+ Result<std::unique_ptr<TableScan>> Build();
+
+ private:
+ /// \brief Reference to the table to scan.
+ const Table& table_;
+ /// \brief column names to project in the scan.
+ std::vector<std::string> column_names_;
+ /// \brief snapshot ID to scan, if specified.
+ std::optional<int64_t> snapshot_id_;
+ /// \brief Context for the scan, including snapshot, schema, and filter.
+ TableScanContext context_;
Review Comment:
TableScanContext will be relied on during subsequent plan files
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2185507366
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+namespace {
+/// \brief Use indexed data structures for efficient lookups
+class DeleteFileIndex {
+ public:
+ /// \brief Build the index from a list of manifest entries.
+ explicit DeleteFileIndex(const std::vector<std::unique_ptr<ManifestEntry>>& entries) {
+ for (const auto& entry : entries) {
+ const int64_t seq_num =
+ entry->sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ sequence_index.emplace(seq_num, entry.get());
+ }
+ }
+
+ /// \brief Find delete files that match the sequence number of a data entry.
+ std::vector<ManifestEntry*> FindRelevantEntries(const ManifestEntry& data_entry) const {
+ std::vector<ManifestEntry*> relevant_deletes;
+
+ // Use lower_bound for efficient range search
+ auto data_sequence_number =
+ data_entry.sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ for (auto it = sequence_index.lower_bound(data_sequence_number);
+ it != sequence_index.end(); ++it) {
Review Comment:
Higher or equal for positional deletes: https://iceberg.apache.org/spec/#scan-planning
##########
src/iceberg/table.cc:
##########
@@ -107,4 +108,8 @@ const std::vector<SnapshotLogEntry>& Table::history() const {
const std::shared_ptr<FileIO>& Table::io() const { return io_; }
+std::unique_ptr<TableScanBuilder> Table::NewScan() const {
+ return std::make_unique<TableScanBuilder>(metadata_, io_);
Review Comment:
How about passing in the `Table` instead? That has all the metadata, and also the `io`
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+namespace {
+/// \brief Use indexed data structures for efficient lookups
+struct DeleteFileIndex {
+ /// \brief Index by sequence number for quick filtering
+ std::multimap<int64_t, ManifestEntry*> sequence_index;
+
+ /// \brief Build the index from a list of manifest entries.
+ void BuildIndex(const std::vector<std::unique_ptr<ManifestEntry>>& entries) {
+ sequence_index.clear();
+
+ for (const auto& entry : entries) {
+ const int64_t seq_num =
+ entry->sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ sequence_index.emplace(seq_num, entry.get());
+ }
+ }
+
+ /// \brief Find delete files that match the sequence number of a data entry.
+ std::vector<ManifestEntry*> FindRelevantEntries(const ManifestEntry& data_entry) const {
+ std::vector<ManifestEntry*> relevant_deletes;
+
+ // Use lower_bound for efficient range search
+ auto data_sequence_number =
+ data_entry.sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ for (auto it = sequence_index.lower_bound(data_sequence_number);
+ it != sequence_index.end(); ++it) {
+ // Additional filtering logic here
Review Comment:
This differs per equality and positional deletes. I think there is a pretty good overview here: https://iceberg.apache.org/spec/#scan-planning
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,285 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+namespace {
+/// \brief Use indexed data structures for efficient lookups
+class DeleteFileIndex {
+ public:
+ /// \brief Build the index from a list of manifest entries.
+ explicit DeleteFileIndex(const std::vector<std::unique_ptr<ManifestEntry>>& entries) {
+ for (const auto& entry : entries) {
+ const int64_t seq_num =
+ entry->sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ sequence_index.emplace(seq_num, entry.get());
+ }
+ }
+
+ /// \brief Find delete files that match the sequence number of a data entry.
+ std::vector<ManifestEntry*> FindRelevantEntries(const ManifestEntry& data_entry) const {
+ std::vector<ManifestEntry*> relevant_deletes;
+
+ // Use lower_bound for efficient range search
+ auto data_sequence_number =
+ data_entry.sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ for (auto it = sequence_index.lower_bound(data_sequence_number);
+ it != sequence_index.end(); ++it) {
+ // Additional filtering logic here
+ relevant_deletes.push_back(it->second);
+ }
+
+ return relevant_deletes;
+ }
+
+ private:
+ /// \brief Index by sequence number for quick filtering
+ std::multimap<int64_t, ManifestEntry*> sequence_index;
+};
+
+/// \brief Get matched delete files for a given data entry.
+std::vector<std::shared_ptr<DataFile>> GetMatchedDeletes(
+ const ManifestEntry& data_entry, const DeleteFileIndex& delete_file_index) {
+ const auto relevant_entries = delete_file_index.FindRelevantEntries(data_entry);
+ std::vector<std::shared_ptr<DataFile>> matched_deletes;
+ if (relevant_entries.empty()) {
+ return matched_deletes;
+ }
+
+ matched_deletes.reserve(relevant_entries.size());
+ for (const auto& delete_entry : relevant_entries) {
+ // TODO(gty404): check if the delete entry contains the data entry's file path
+ matched_deletes.emplace_back(delete_entry->data_file);
Review Comment:
Without this filter, it will likely explode the number of relevant entries for each of the data files.
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,285 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+namespace {
+/// \brief Use indexed data structures for efficient lookups
+class DeleteFileIndex {
+ public:
+ /// \brief Build the index from a list of manifest entries.
+ explicit DeleteFileIndex(const std::vector<std::unique_ptr<ManifestEntry>>& entries) {
Review Comment:
Do we want to add this right away, or defer this to a later PR? Previously at PyIceberg we threw an exception when we encountered a positional or equality delete.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
Re: [PR] feat: basic table scan planning [iceberg-cpp]
Posted by "gty404 (via GitHub)" <gi...@apache.org>.
gty404 commented on code in PR #112:
URL: https://github.com/apache/iceberg-cpp/pull/112#discussion_r2182246071
##########
src/iceberg/table_scan.cc:
##########
@@ -0,0 +1,294 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/table_scan.h"
+
+#include <algorithm>
+#include <ranges>
+
+#include "iceberg/manifest_entry.h"
+#include "iceberg/manifest_list.h"
+#include "iceberg/manifest_reader.h"
+#include "iceberg/schema.h"
+#include "iceberg/schema_field.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+namespace {
+/// \brief Use indexed data structures for efficient lookups
+class DeleteFileIndex {
+ public:
+ /// \brief Build the index from a list of manifest entries.
+ explicit DeleteFileIndex(const std::vector<std::unique_ptr<ManifestEntry>>& entries) {
+ for (const auto& entry : entries) {
+ const int64_t seq_num =
+ entry->sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ sequence_index.emplace(seq_num, entry.get());
+ }
+ }
+
+ /// \brief Find delete files that match the sequence number of a data entry.
+ std::vector<ManifestEntry*> FindRelevantEntries(const ManifestEntry& data_entry) const {
+ std::vector<ManifestEntry*> relevant_deletes;
+
+ // Use lower_bound for efficient range search
+ auto data_sequence_number =
+ data_entry.sequence_number.value_or(TableMetadata::kInitialSequenceNumber);
+ for (auto it = sequence_index.lower_bound(data_sequence_number);
+ it != sequence_index.end(); ++it) {
+ // Additional filtering logic here
+ relevant_deletes.push_back(it->second);
+ }
+
+ return relevant_deletes;
+ }
+
+ private:
+ /// \brief Index by sequence number for quick filtering
+ std::multimap<int64_t, ManifestEntry*> sequence_index;
+};
+
+/// \brief Get matched delete files for a given data entry.
+std::vector<std::shared_ptr<DataFile>> GetMatchedDeletes(
+ const ManifestEntry& data_entry, const DeleteFileIndex& delete_file_index) {
+ const auto relevant_entries = delete_file_index.FindRelevantEntries(data_entry);
+ std::vector<std::shared_ptr<DataFile>> matched_deletes;
+ if (relevant_entries.empty()) {
+ return matched_deletes;
+ }
+
+ matched_deletes.reserve(relevant_entries.size());
+ for (const auto& delete_entry : relevant_entries) {
+ // TODO(gty404): check if the delete entry contains the data entry's file path
+ matched_deletes.emplace_back(delete_entry->data_file);
+ }
+ return matched_deletes;
+}
+} // namespace
+
+// implement FileScanTask
+FileScanTask::FileScanTask(std::shared_ptr<DataFile> file,
+ std::vector<std::shared_ptr<DataFile>> delete_files,
+ int64_t start, int64_t length,
+ std::shared_ptr<Expression> residual)
+ : data_file_(std::move(file)),
+ delete_files_(std::move(delete_files)),
+ start_(start),
+ length_(length),
+ residual_(std::move(residual)) {}
Review Comment:
Yes, the start and end are not required, I will remove them from FileScanTask.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org