You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lx...@apache.org on 2026/09/08 14:40:05 UTC
(paimon-cpp) branch main updated: feat(format-table): support reading and writing format tables (#222)
This is an automated email from the ASF dual-hosted git repository.
lxy-9602 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-cpp.git
The following commit(s) were added to refs/heads/main by this push:
new 5566b64a feat(format-table): support reading and writing format tables (#222)
5566b64a is described below
commit 5566b64a929ce25c1745252f452ce648c39e0197
Author: Nicholas Jiang <pr...@163.com>
AuthorDate: Tue Sep 8 22:39:58 2026 +0800
feat(format-table): support reading and writing format tables (#222)
---
docs/source/api.rst | 1 +
docs/source/api/format_table.rst | 52 +
docs/source/user_guide.rst | 1 +
docs/source/user_guide/format_table.rst | 351 ++
include/paimon/api.h | 1 +
include/paimon/catalog/catalog.h | 28 +
include/paimon/commit_context.h | 21 +-
include/paimon/defs.h | 23 +
include/paimon/read_context.h | 20 +-
include/paimon/scan_context.h | 20 +-
include/paimon/table/format/format_table.h | 207 +
include/paimon/table/source/split.h | 9 +-
include/paimon/table/source/table_scan.h | 13 +
include/paimon/write_context.h | 22 +-
src/paimon/CMakeLists.txt | 19 +
src/paimon/common/defs.cpp | 6 +
.../common/reader/data_file_reader_factory.cpp | 96 +
.../common/reader/data_file_reader_factory.h | 95 +
.../common/utils/binary_row_partition_computer.cpp | 15 +-
.../common/utils/binary_row_partition_computer.h | 15 +
.../utils/binary_row_partition_computer_test.cpp | 38 +
src/paimon/common/utils/hadoop_compression.h | 121 +
.../common/utils/hadoop_compression_test.cpp | 82 +
src/paimon/common/utils/string_utils.cpp | 2 +-
src/paimon/common/utils/string_utils.h | 4 +-
src/paimon/core/catalog/catalog.cpp | 19 +
src/paimon/core/catalog/catalog_utils.cpp | 49 +
src/paimon/core/catalog/catalog_utils.h | 33 +
src/paimon/core/catalog/file_system_catalog.cpp | 7 +
src/paimon/core/catalog/file_system_catalog.h | 7 +
.../core/catalog/file_system_catalog_test.cpp | 45 +
src/paimon/core/core_options.cpp | 46 +
src/paimon/core/core_options.h | 17 +
src/paimon/core/core_options_test.cpp | 57 +
src/paimon/core/operation/abstract_split_read.cpp | 62 +-
src/paimon/core/operation/abstract_split_read.h | 5 +
src/paimon/core/operation/commit_context.cpp | 32 +-
src/paimon/core/operation/file_store_commit.cpp | 94 +-
src/paimon/core/operation/file_store_write.cpp | 57 +-
src/paimon/core/operation/read_context.cpp | 44 +-
src/paimon/core/operation/scan_context.cpp | 41 +-
src/paimon/core/operation/write_context.cpp | 40 +-
src/paimon/core/options/table_type.h | 98 +
src/paimon/core/options/table_type_test.cpp | 93 +
src/paimon/core/schema/schema_manager.cpp | 5 +-
src/paimon/core/schema/schema_validation.cpp | 186 +-
src/paimon/core/schema/schema_validation.h | 48 +
.../core/table/format/format_commit_message.h | 69 +
src/paimon/core/table/format/format_data_split.h | 79 +
.../core/table/format/format_file_listing.cpp | 108 +
src/paimon/core/table/format/format_file_listing.h | 72 +
.../core/table/format/format_file_listing_test.cpp | 192 +
.../core/table/format/format_file_naming.cpp | 98 +
src/paimon/core/table/format/format_file_naming.h | 77 +
.../core/table/format/format_file_naming_test.cpp | 111 +
.../core/table/format/format_path_validation.cpp | 368 ++
.../core/table/format/format_path_validation.h | 86 +
src/paimon/core/table/format/format_table.cpp | 211 +
.../core/table/format/format_table_commit.cpp | 390 ++
src/paimon/core/table/format/format_table_commit.h | 99 +
.../format/format_table_file_store_commit.cpp | 174 +
.../table/format/format_table_file_store_commit.h | 102 +
.../table/format/format_table_file_store_write.cpp | 95 +
.../table/format/format_table_file_store_write.h | 70 +
.../core/table/format/format_table_loader.cpp | 86 +
src/paimon/core/table/format/format_table_loader.h | 71 +
src/paimon/core/table/format/format_table_read.cpp | 617 +++
src/paimon/core/table/format/format_table_read.h | 111 +
src/paimon/core/table/format/format_table_scan.cpp | 288 ++
src/paimon/core/table/format/format_table_scan.h | 116 +
src/paimon/core/table/format/format_table_test.cpp | 5060 ++++++++++++++++++++
.../core/table/format/format_table_write.cpp | 559 +++
src/paimon/core/table/format/format_table_write.h | 78 +
.../core/table/format/lazy_concat_batch_reader.cpp | 129 +
.../core/table/format/lazy_concat_batch_reader.h | 85 +
.../table/format/lazy_concat_batch_reader_test.cpp | 291 ++
.../core/table/sink/commit_message_serializer.cpp | 8 +
src/paimon/core/table/source/split.cpp | 7 +
src/paimon/core/table/source/table_read.cpp | 59 +-
src/paimon/core/table/source/table_scan.cpp | 93 +-
src/paimon/core/table/table.cpp | 7 +
src/paimon/core/table/table_test.cpp | 34 +
src/paimon/core/utils/partition_path_utils.cpp | 77 +-
src/paimon/core/utils/partition_path_utils.h | 36 +-
.../core/utils/partition_path_utils_test.cpp | 84 +
src/paimon/format/orc/orc_format_writer.cpp | 11 +
src/paimon/rest/rest_catalog.cpp | 24 +
src/paimon/rest/rest_catalog.h | 6 +
src/paimon/rest/rest_catalog_test.cpp | 217 +-
src/paimon/rest/rest_messages.cpp | 5 +
src/paimon/rest/rest_messages_test.cpp | 9 +
91 files changed, 12786 insertions(+), 130 deletions(-)
diff --git a/docs/source/api.rst b/docs/source/api.rst
index 2f6538a2..93ac4ce0 100644
--- a/docs/source/api.rst
+++ b/docs/source/api.rst
@@ -27,6 +27,7 @@ API Reference
api/commit
api/scan
api/read
+ api/format_table
api/predicate
api/file_format
api/file_system
diff --git a/docs/source/api/format_table.rst b/docs/source/api/format_table.rst
new file mode 100644
index 00000000..1ec96437
--- /dev/null
+++ b/docs/source/api/format_table.rst
@@ -0,0 +1,52 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+
+.. http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied. See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+============
+Format Table
+============
+
+.. _cpp-api-format-table:
+
+Table
+=====
+
+.. doxygenclass:: paimon::FormatTable
+ :members:
+ :undoc-members:
+
+Reading and writing a format table go through the generic entry points every other table uses:
+:cpp:class:`paimon::TableScan`, :cpp:class:`paimon::TableRead`, :cpp:class:`paimon::FileStoreWrite`
+and :cpp:class:`paimon::FileStoreCommit`. See :doc:`../user_guide/format_table`.
+
+Contexts
+========
+
+Each of those entry points builds its context from a table path, and the schema under that path
+says what kind of table it is. A caller that already holds a ``FormatTable`` hands it over instead,
+through the constructor each context builder has for one:
+
+* :cpp:class:`paimon::ScanContextBuilder`\ ``(const std::shared_ptr<FormatTable>&)``
+* :cpp:class:`paimon::ReadContextBuilder`\ ``(const std::shared_ptr<FormatTable>&)``
+* :cpp:class:`paimon::WriteContextBuilder`\ ``(const std::shared_ptr<FormatTable>&)``
+* :cpp:class:`paimon::CommitContextBuilder`\ ``(const std::shared_ptr<FormatTable>&)``
+
+This is the only way to reach a format table whose schema does not live under its own location,
+such as one a REST catalog serves: nothing under the location says that it is a format table, nor
+that what sits below it is data rather than metadata. The table knows both, so a setting that would
+answer either question again - ``SetTableSchema()``, ``WithFileSystem()`` and a branch among them -
+is refused by these builders rather than quietly ignored. Options given at the call still win over
+the ones the table carries, as they do everywhere else.
diff --git a/docs/source/user_guide.rst b/docs/source/user_guide.rst
index fdee89d9..c03efd88 100644
--- a/docs/source/user_guide.rst
+++ b/docs/source/user_guide.rst
@@ -33,6 +33,7 @@ User Guide
user_guide/data_types
user_guide/primary_key_table
user_guide/append_only_table
+ user_guide/format_table
user_guide/system_tables
user_guide/write
user_guide/commit
diff --git a/docs/source/user_guide/format_table.rst b/docs/source/user_guide/format_table.rst
new file mode 100644
index 00000000..b52bd5d3
--- /dev/null
+++ b/docs/source/user_guide/format_table.rst
@@ -0,0 +1,351 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+
+.. http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied. See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+.. Ported from the Paimon documentation:
+.. https://github.com/apache/paimon/blob/master/docs/docs/concepts/rest/tables.mdx
+
+.. default-domain:: cpp
+.. highlight:: cpp
+
+Format Table
+============
+A format table is a directory that holds multiple files of the same format. It carries no
+snapshots and no manifests: the files in the directory are the table, so reading it lists
+directories and writing to it adds files. A table is a format table when its ``type`` option is
+``format-table``; ``file.format`` then names the format of every file in it, which here is
+``parquet`` or ``orc``.
+
+A partitioned format table uses the standard Hive directory layout, and its partitions are
+discovered from that layout rather than from metadata. By default a partition directory is named
+``key=value``; setting ``format-table.partition-path-only-value`` names it by the value alone.
+
+Because a directory of plain files records no row identity, a format table only accepts inserts.
+Reads still carry the leading ``_VALUE_KIND`` field every ``BatchReader`` promises, so an engine
+that reads a batch by field index sees the same layout it does for a managed table; every row of a
+format table is an insert.
+
+Reading and writing
+-------------------
+A format table is not served through :cpp:func:`Catalog::GetTable`, which describes a managed
+table; use :cpp:func:`Catalog::GetFormatTable` instead.
+
+Reading and writing go through the entry points every other table uses: ``TableScan::Create``,
+``TableRead::Create``, ``FileStoreWrite::Create`` and ``FileStoreCommit::Create``, each from its
+usual context builder. Each reads the table's schema - from under the table path, or from the one
+the context carries - and dispatches to a format table when its ``type`` says so.
+
+That path needs a schema under the table's own location, which is what every table a file system
+catalog serves has. A table loaded from a catalog that keeps the schema elsewhere, such as the
+REST catalog, has none: nothing under the location says that it is a format table, nor that
+``schema`` and ``branch`` below it are data rather than metadata. Such a table is read and written
+by handing the loaded ``FormatTable`` to the context instead of a path - every context builder has
+a constructor taking one - which is what Java does through ``FormatTable.newReadBuilder()`` and
+``newBatchWriteBuilder()``:
+
+.. code-block:: cpp
+
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<paimon::FormatTable> table,
+ catalog->GetFormatTable(paimon::Identifier("db", "tbl")));
+ paimon::ScanContextBuilder scan_builder(table);
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<paimon::ScanContext> scan_context,
+ scan_builder.Finish());
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<paimon::TableScan> scan,
+ paimon::TableScan::Create(std::move(scan_context)));
+
+The table already carries its schema and the file system it was loaded through, so a context built
+from one refuses ``SetTableSchema()``, ``WithFileSystem()``,
+``WithFileSystemSchemeToIdentifierMap()`` and a branch rather than quietly ignoring them. Options
+given at the call still win over the ones the schema stored, as they do everywhere else.
+
+A batch comes back in the table's column order, or the projection's when the read names one,
+behind the ``_VALUE_KIND`` field every ``BatchReader`` carries. Which columns a *file* is asked for
+and in what order is its own business: they are matched to the table's by name, as Java matches
+them through ``ParquetReaderFactory`` and orc's schema evolution, so a file that stores them in
+another order is read in the order it stores them, and one holding columns the table does not name
+is read past. A column the file does not hold is filled with nulls, which is how a file written
+before a column was added is still read - unless the table declares that column ``NOT NULL``, when
+the file is refused instead, as Java's ``VectorizedParquetRecordReader`` refuses it. Only the
+columns a read actually asks a file for are held to the table's schema: a partition column is
+rebuilt from the directory name whether or not the file stores one of its own, and a column
+outside the projection is never read, which is how Java scopes the same checks.
+
+``FormatTable`` is the only format-table type in the public API. The classes behind it -
+``FormatTableScan``, ``FormatTableRead``, ``FormatTableWrite``, ``FormatTableCommit``,
+``FormatDataSplit`` and ``FormatCommitMessage`` - are implementation details under ``src`` and are
+named here only to describe what happens. A caller never needs them: a plan comes back as
+``Plan``, a split as ``Split``, and a commit message as ``CommitMessage``.
+
+A catalog that can load a format table itself overrides ``Catalog::LoadFormatTable()``, the
+protected hook :cpp:func:`Catalog::GetFormatTable` calls. The file system catalog uses it to say
+its metadata lives under the table location, and the REST catalog to take the location and the
+schema from one response instead of two that could disagree. A catalog that does not override it
+still serves format tables, by reading the location and the schema through the virtuals every
+catalog has.
+
+``TableScan::ListPartitions()`` lists the partitions a scan can see. A format table answers it by
+listing directories; every other table type returns ``NotImplemented`` for now.
+
+Not all of the generic interfaces fit. ``FileStoreCommit`` is mostly about snapshots and manifests -
+expiring them, rolling back to one, filtering by a commit identifier recorded in one - and a format
+table keeps none of that state, so those calls are refused rather than quietly doing nothing.
+``FileStoreWrite::Compact()`` is refused for the same reason, and both write and commit take batch
+writes only, since there is no snapshot to record a commit identifier or a watermark in.
+``TableScan`` takes a partition filter and a limit; a predicate or a bucket filter is refused. Java
+refuses a predicate from ``FormatTableScan.withFilter`` too, but ``FormatReadBuilder.newScan()``
+splits one first and hands the partition half to the scan, so there a predicate over partition
+columns still prunes directories. See the limits below.
+
+Options given at the call win over the ones the schema stored, as they do for every other table -
+except ``type``, which is structural and is read from the schema alone, so one read or write
+cannot decide what kind of table this is. The merged result is validated, not the schema's own
+options, so an option a format table refuses - ``metastore.partitioned-table``, a file format
+nothing here can read - is refused wherever it comes from rather than dropped in silence.
+
+A setting the format path cannot act on is refused by name rather than quietly dropped:
+
+* ``ReadContextBuilder::SetReadSchema()``. A projected read schema can rename a column, prune a
+ nested one and give it metadata of its own; a format table's projection is a list of top-level
+ names, so name the columns instead.
+* ``WithStreamingMode()`` on a scan or a write, a global index result on a scan, and a real-time
+ context on a scan, a read or a write: a format table has no snapshots, no real-time store and no
+ index.
+* ``WriteContextBuilder::WithWriteSchema()``, which names a subset of the columns to write.
+* ``WriteContextBuilder::WithWriteId()``, which prefixes a postpone-bucket writer's files so one
+ compaction reader can put them back in order; a format table has no buckets.
+* ``CommitContextBuilder::IgnoreEmptyCommit(false)``, ``UseRESTCatalogCommit(true)`` and
+ ``AppendCommitCheckConflict(true)``. Keeping an empty commit means writing a snapshot that adds
+ no files, a rest-catalog commit sends that snapshot to a catalog, and the conflict check reads
+ the manifests of concurrent commits - none of which exist here. Each is refused only when set
+ away from its default, so an ordinary commit is unaffected.
+* the ``branch`` option, whatever names it: a format table keeps no metadata to branch, and its
+ data is the files under its one location, which is where a read or a write would go whichever
+ branch was asked for. It is refused wherever it comes from - the schema, a catalog that parsed
+ ``tbl$branch_dev``, or the options of a single call - and so is ``WithBranch()`` on a context
+ built from a format table. A branch quietly ignored would be the wrong answer everywhere and a
+ destructive one for an overwrite, which would clear the data it was told not to touch. The
+ value naming the main branch is the default and is left alone.
+* A scan predicate or bucket filter, as above, and more than one partition filter: a scan descends
+ one directory layout, so it takes the values of a single partition rather than a set of them.
+
+What a data file is opened with is not one of the refusals. ``EnablePrefetch()``, the read-ahead
+cache and its ``CacheConfig``, and the ``Cache`` a read carries all apply, because a format table
+opens its files through the same component the managed table path opens its own with. What differs
+between the two paths is which files there are and how a row is put back together, not how a file
+is read.
+
+Some settings are not refused because they describe machinery the format path never reaches, and
+refusing them would refuse the defaults: ``EnableMultiThreadRowToBatch()`` on a read, a write's
+temporary directory and spill configuration, and ``WithIgnoreNumBucketCheck()`` and
+``WithIgnorePreviousFiles()`` on a write. They have no effect here: a format read hands out the
+batches parquet or orc already produced rather than assembling them from rows, a format write
+buffers in memory and never spills, and a table with no buckets has no bucket count to check and
+no previous files to read back.
+
+A write is two-phase, since a directory has no metadata to switch atomically: a file is written
+into a ``_temporary`` directory beside where it will end up, under a hidden name of its own, and
+only the commit renames it into place. That is the layout Java Paimon's
+``RenamingTwoPhaseOutputStream`` stages under. The directory and the name are both hidden, the
+convention a Hive-style directory uses for output that is not committed table data, and what a scan
+of this table skips. The ``_temporary`` directory is shared with every other writer
+of the same table and is left behind after a commit.
+
+An overwriting commit clears what it replaces before publishing anything. What it replaces is the
+static partition when one is given - the partitions below that prefix, whether or not this commit
+wrote to them - and otherwise the partitions the commit actually writes to, when
+``dynamic-partition-overwrite`` is on, its default, and the table is partitioned. With the option
+off, or for an unpartitioned table, it replaces the whole table instead, which is the condition
+Java's ``FormatTableCommit`` and a managed table commit both apply. An overwrite that replaces the
+table empties it even when it publishes no file, so a statement whose query returns nothing still
+clears what was there.
+
+A plan is in-memory only. ``FormatDataSplit`` has no serialized form - ``Split::Serialize()``
+refuses it - and neither has ``FormatCommitMessage``: a format table's plan has no cross-runtime
+encoding, so plan, read and commit within one process.
+
+A ``FormatTableWrite`` and a ``FormatTableCommit`` are each driven by one thread, but separate
+ones may fill and add to a table at once: each write stages its files under a uuid of its own, and
+each commit publishes only the files its own messages name. Two *overwriting* commits over the
+same directory race, since an overwrite clears what is committed there before publishing anything.
+A ``FormatTableScan`` may be shared, since planning leaves it as it was.
+
+``TableRead::CreateCountReader()`` is not implemented for a format table, so counting its rows
+means reading them. That is a gap here rather than something the layout forces: ``parquet`` and
+``orc`` both record a row count in their own footer.
+
+A writer starts a new file once the one it is filling reaches ``target-file-row-num`` rows or
+``target-file-size`` bytes. Both are checked between batches rather than between rows, because a
+batch is the unit this API writes in, so a file may pass either target by up to one batch. Java
+checks the row count on every row and the size every thousand rows, and its files therefore sit
+closer to the target.
+
+Aborting a write
+----------------
+``FormatTableWrite::Abort()`` removes the files the write staged. It is the one call still allowed
+after ``PrepareCommit()``, and that is what it is for: a write dropped *before* preparing clears
+its staged files from its own destructor, so only a commit that is prepared and then abandoned
+needs it.
+
+Path containment is checked on the path text, which stops a ``..`` from leaving the table but not
+a symbolic link pointing out of it - the same as Java's own local file system behaviour. A
+relative local location is resolved against the working directory first, as the local file system
+resolves one while opening a file, so a table located at ``./table`` and the absolute paths its
+own listing hands back are recognised as the same place. ``FormatTable::Location()`` still reports
+the location as it was given.
+
+``FormatTableCommit::Abort()`` does the same for the messages a commit was given. **Neither undoes
+a commit that succeeded**: once a file has been renamed into place it is no longer staged, and
+nothing here will take it back. Java's committer removes the published path as well as the staged
+one, which matters there because a commit publishes file by file with nothing watching; here a
+commit that fails part way takes back every target it tried to publish before it returns, so an
+abort is left with the staged files alone. That includes the target of the rename the commit
+failed on, which a store may have produced before failing to say so - on an object store a rename
+is a copy followed by a delete, and a request that took effect but lost its response looks like
+one that did nothing. The target is removed on the strength of it belonging to this write: its
+name carries the uuid of the write that staged it, and no two messages of one commit may name the
+same target. An overwriting commit is the exception: it has already deleted what it replaces, so
+taking the published files back too would leave neither the old rows nor the new, and they are
+kept. Java preserves them for the same reason.
+
+Both are best effort and never fail here, so a warning in the log is the only signal that a file
+could not be removed - and a published target left behind is one a scan goes on to read. Java's
+``abort`` instead collects its failures and reports them.
+
+Give ``FormatTableCommit`` only the messages this job's own writers produced. A message names a
+staged file by path, and a commit can tell that the path belongs to this table, sits in the
+partition the message declares, and is staged rather than already published - not whose staged file
+it is. A well-formed message from somewhere else is published, or discarded by ``Abort()``, like
+any other.
+
+Relationship to Java Paimon
+---------------------------
+Java serves format tables from a Hive or REST catalog, which holds the schema. This implementation
+also serves them from a file system catalog, which keeps the schema under the table directory - an
+extension Java does not have. Only for such a table are the ``schema`` and ``branch`` directories
+below the location treated as metadata rather than as data.
+
+A file system catalog keeps a table's schema in ``schema`` and its branches in ``branch`` below
+the table location, so under ``format-table.partition-path-only-value`` the first partition value
+may not be ``schema`` or ``branch``: the directory a write would use is the one holding the
+table's own metadata. Such a write is refused, as is an overwrite naming that partition - which
+would otherwise delete the schema. A table served from a REST or Hive catalog keeps its schema
+elsewhere, so there these are ordinary partition values and are read and written like any other.
+
+Under that same layout a partition value may not start with ``_`` or ``.`` either, whichever
+catalog serves the table: the value is the whole directory name, and a scan skips every hidden
+name. Java writes such a directory and then cannot read it back; here the write is refused
+instead. The one exception is the value standing for a null partition, ``partition.default-name``,
+which the scan reads at a partition level by design. Under the ``key=value`` layout the question
+does not arise, since the key in front of the value keeps the directory name visible.
+
+A few smaller differences come from this library's own conventions:
+
+* a write takes one partition per batch: the batch declares it through
+ ``RecordBatchBuilder::SetPartition()`` and every row of that batch is stored under it. Java
+ routes row by row, so one write call there may land in any number of partitions;
+* a write takes its partition from ``RecordBatchBuilder::SetPartition()`` rather than from the
+ rows. A row's own partition column is not checked against the declaration and never reads back:
+ a format table's partition columns are rebuilt from the directory name, as a managed table's are
+ from the manifest, and the managed write path takes ``RecordBatch::GetPartition()`` on the same
+ terms. The values arrive as text. They are still read into their column types and rendered back
+ out before anything is named after them - the round trip Java's writer makes when it renders a
+ partition out of the row it is writing, through the partition computer its
+ ``FileStorePathFactory`` holds. The table therefore decides the directory name and the commit
+ message, not the spelling the caller used: with ``partition.legacy-name`` on, its default, a
+ ``DATE`` partition is written as its day count whether the caller wrote ``19723`` or
+ ``2024-01-01``, and as ``YYYY-MM-DD`` when the option is off. A value that cannot be read into
+ its column type is refused. ``FormatTableCommit``'s static partition is *not* put through that
+ round trip and is used as given, which is what Java's ``FormatTableCommit.buildPartitionPath``
+ does with it too;
+* a commit message carries the partition its file belongs to, and a commit checks that it agrees
+ with the directory the file sits in. Java's message carries none and derives the partition from
+ the committer's target path, so the two cannot disagree there. A message here reaches the commit
+ through the base ``CommitMessage`` type, so the value is checked rather than trusted;
+* a projection that names the same column twice is rejected when the read is built. Java reads
+ such a column once per entry;
+* a row limit is the caller's. ``FormatTableScan`` takes one, but a format table records no row
+ counts, so only a non-positive limit prunes anything - it plans nothing at all. A positive one
+ neither drops splits nor bounds the reader ``FormatTableRead`` hands out, so the caller stops
+ calling ``NextBatch()`` once it has enough. Java wraps its reader in a ``LimitRecordReader``.
+
+Current limits
+--------------
+The layout, the option precedence and the read, write and overwrite semantics described above
+follow Java Paimon's. What this implementation covers is nonetheless a subset of what Java's
+format table does; it does not yet support:
+
+* the ``csv``, ``json``, ``text`` and ``mosaic`` file formats, leaving ``parquet`` and ``orc``.
+ The first three are line-delimited text in Java, which shares one line-reading layer between
+ them; this library has no text file format at all, so the first of them to be added has to bring
+ that layer with it. ``mosaic`` is none of those: it has a reader and a writer of its own, which
+ this library builds under ``PAIMON_ENABLE_MOSAIC``, but a format table does not reach them yet;
+* cutting one large data file into byte ranges so that several readers share it. Java does this
+ for an uncompressed ``csv`` or ``json`` file written with the default line delimiter, and for no
+ other format - not for ``text`` or ``mosaic`` either; ``parquet`` and ``orc`` each record where
+ their own row groups and stripes begin, and a reader handed a byte range of one would have to
+ find that out for itself;
+* ``metastore.partitioned-table``, which moves partition visibility into the catalog, and the
+ Hive partition sync that goes with it;
+* partition filters beyond equality on partition values, where Java accepts a full predicate. The
+ values are compared through their column type, as Java's are, but discovery here lists one
+ directory level at a time and tests each name's value, while Java turns a leading run of
+ equality constraints into a path and starts listing below it; a table with many partitions
+ therefore costs more listings here than in Java;
+* ``scan.ignore-corrupt-files`` and ``scan.ignore-lost-files``, which are not implemented: a
+ corrupt or missing data file fails the read rather than being skipped;
+* reading a column whose type in the file is not the type the table declares. A column is matched
+ by name and then read as the table declares it, so a file storing it as some other type fails
+ the read rather than being converted to that type. Java's orc reader converts a few such pairs,
+ through orc's own schema evolution. The same goes for a ``STRUCT`` whose children the file
+ orders differently, which the read refuses rather than putting back in the table's order. A
+ ``TIMESTAMP`` of another unit is the one difference the read carries, and how far depends on
+ the format: an orc file records no unit at all - a value is a seconds-plus-nanoseconds pair,
+ built at whatever precision the read asks for - so any unit is read as the table declares it,
+ while a parquet file records the unit its values are in and only ``MILLI`` under a ``SECOND``
+ column is rescaled, the pair parquet's own round trip needs, every other being refused rather
+ than relabelled;
+* ``FileStoreCommit::TruncateTable()``, which Java implements for a format table by deleting its
+ data files while leaving the partition directories in place, and
+ ``FileStoreCommit::DropPartition()``, which would remove a partition directory outright. An
+ overwrite of a partition with no messages empties it but leaves the directory, which is what
+ Java's ``truncatePartitions()`` does rather than a drop;
+* ``format-table.commit-hive-sync-url``, which registers committed partitions with a Hive
+ metastore;
+* column default values. Java replaces a null in a column whose schema field declares a default
+ with that default as it writes; here the null is written as it came;
+* a table every one of whose columns is a partition column. Java projects the partition columns
+ out of what it writes, leaving files that carry nothing but a row count; here such a schema is
+ refused when the table is created and when it is opened;
+* ``TIMESTAMP``, ``DECIMAL``, ``FLOAT`` and ``DOUBLE`` partition columns, which Java allows. This
+ is a restriction of the whole library rather than of format tables. The types that do work are
+ ``BOOLEAN``, ``TINYINT``, ``SMALLINT``, ``INT``, ``BIGINT``, ``STRING`` and ``DATE`` - the set
+ the managed table path reads and writes partitions in. Any other, ``BINARY`` among them, is
+ refused when the table is created and when it is opened, rather than at the first read or write:
+ validation asks by building the partition computer that does the round trip, so there is one
+ answer rather than a list of types that could fall out of step with it.
+
+``data-file.path-directory`` has no effect here, and none in Java either: Java's format table
+writer builds its paths from the table root rather than from that directory.
+``format-table.implementation`` is honoured by the engines rather than by the table - in Java
+Spark it selects between Paimon's own implementation and the engine's ``FileTable`` - so it has
+no meaning inside this library.
+
+Validation
+----------
+A table Java can serve and this library cannot is refused at creation rather than accepted and
+then found unopenable, whichever catalog it is created through. It can still reach a catalog
+another way - written by Java, or by an older client - so the same checks run again when the
+table is opened.
diff --git a/include/paimon/api.h b/include/paimon/api.h
index f188085d..2f9d474b 100644
--- a/include/paimon/api.h
+++ b/include/paimon/api.h
@@ -36,6 +36,7 @@
#include "paimon/snapshot/snapshot_file_scan.h" // IWYU pragma: export
#include "paimon/statistics_mode.h" // IWYU pragma: export
#include "paimon/status.h" // IWYU pragma: export
+#include "paimon/table/format/format_table.h" // IWYU pragma: export
#include "paimon/table/source/table_read.h" // IWYU pragma: export
#include "paimon/table/source/table_scan.h" // IWYU pragma: export
#include "paimon/write_context.h" // IWYU pragma: export
diff --git a/include/paimon/catalog/catalog.h b/include/paimon/catalog/catalog.h
index 1162521a..f003509b 100644
--- a/include/paimon/catalog/catalog.h
+++ b/include/paimon/catalog/catalog.h
@@ -40,6 +40,7 @@ namespace paimon {
using Instant = std::variant<std::string, int64_t>;
class Database;
+class FormatTable;
class Table;
class View;
class Schema;
@@ -209,6 +210,33 @@ class PAIMON_EXPORT Catalog {
/// snapshot id ascending, or an error status.
virtual Result<std::vector<SnapshotInfo>> ListSnapshots(
const Identifier& identifier, const std::string& branch = "") const = 0;
+
+ /// Gets a format table: a directory of data files laid out like a standard Hive table.
+ ///
+ /// A format table carries no snapshots and no manifests, so it is loaded through its own
+ /// method rather than `GetTable()`. Reading and writing it go through the same `TableScan`,
+ /// `TableRead`, `FileStoreWrite` and `FileStoreCommit` entry points every other table uses:
+ /// hand the table back to `ScanContextBuilder`, `ReadContextBuilder`, `WriteContextBuilder` or
+ /// `CommitContextBuilder`, each of which takes one.
+ ///
+ /// @param identifier Identifier of the table to get.
+ /// @return A result containing the format table, or an error status if the table does not
+ /// exist or its `type` option is not `format-table`.
+ Result<std::shared_ptr<FormatTable>> GetFormatTable(const Identifier& identifier) const;
+
+ protected:
+ /// Loads `identifier` as a format table, which is what `GetFormatTable()` hands back.
+ ///
+ /// The default reads the location and the schema through the virtuals above - two requests
+ /// that can disagree - and treats every directory below the location as table content. An
+ /// override answers both questions directly: whether location and schema come back in one
+ /// round trip, and whether this catalog keeps the table's metadata under the table path.
+ ///
+ /// @param identifier Identifier of the table to load.
+ /// @return A result containing the format table, or an error status if the table does not
+ /// exist or its `type` option is not `format-table`.
+ virtual Result<std::shared_ptr<FormatTable>> LoadFormatTable(
+ const Identifier& identifier) const;
};
} // namespace paimon
diff --git a/include/paimon/commit_context.h b/include/paimon/commit_context.h
index 91acb1c1..e0b3e4b1 100644
--- a/include/paimon/commit_context.h
+++ b/include/paimon/commit_context.h
@@ -29,6 +29,7 @@
namespace paimon {
class Executor;
+class FormatTable;
class MemoryPool;
/// `CommitContext` is some configuration for commit operations.
@@ -43,7 +44,8 @@ class PAIMON_EXPORT CommitContext {
bool append_commit_check_conflict, const std::shared_ptr<MemoryPool>& memory_pool,
const std::shared_ptr<Executor>& executor,
const std::shared_ptr<FileSystem>& specific_file_system,
- const std::map<std::string, std::string>& options);
+ const std::map<std::string, std::string>& options,
+ const std::shared_ptr<FormatTable>& format_table);
~CommitContext();
const std::string& GetRootPath() const {
@@ -82,6 +84,12 @@ class PAIMON_EXPORT CommitContext {
return options_;
}
+ /// The format table this context was built from, or null when it names a table path and the
+ /// schema under that path says what kind of table it is.
+ const std::shared_ptr<FormatTable>& GetFormatTable() const {
+ return format_table_;
+ }
+
private:
std::string root_path_;
std::string commit_user_;
@@ -92,6 +100,7 @@ class PAIMON_EXPORT CommitContext {
std::shared_ptr<Executor> executor_;
std::shared_ptr<FileSystem> specific_file_system_;
std::map<std::string, std::string> options_;
+ std::shared_ptr<FormatTable> format_table_;
};
/// `CommitContextBuilder` used to build a `CommitContext`, has input validation.
@@ -102,6 +111,16 @@ class PAIMON_EXPORT CommitContextBuilder {
/// @param commit_user The user identifier for the commit operation.
CommitContextBuilder(const std::string& root_path, const std::string& commit_user);
+ /// Constructs a `CommitContextBuilder` for a format table that is already loaded: the only way
+ /// to commit to one whose schema lives in a metastore rather than under its location, such as
+ /// a table a REST catalog serves. The table carries what such a location does not say, so
+ /// `WithFileSystem()` is refused here rather than ignored.
+ ///
+ /// There is no commit user: a format table keeps no snapshot to record one in.
+ ///
+ /// @param table The format table to commit to, as `Catalog::GetFormatTable()` hands it back.
+ explicit CommitContextBuilder(const std::shared_ptr<FormatTable>& table);
+
~CommitContextBuilder();
/// Set a configuration options map to set some option entries which are not defined in the
diff --git a/include/paimon/defs.h b/include/paimon/defs.h
index e819c85d..8ecda26a 100644
--- a/include/paimon/defs.h
+++ b/include/paimon/defs.h
@@ -118,10 +118,33 @@ struct PAIMON_EXPORT Options {
/// "page-size" - Memory page size, default value 64 kb.
static const char PAGE_SIZE[];
+ /// "type" - Type of the table. Default value is "table" (a managed paimon table);
+ /// "format-table" declares a directory of plain data files laid out like a Hive table.
+ static const char TYPE[];
+
/// "file.format" - Specify the message format of data files.
/// Default value is parquet.
static const char FILE_FORMAT[];
+ /// "format-table.file.compression" - File compression of a format table. It is consulted
+ /// after "file.compression" and before the bare "compression" key an engine's own writer
+ /// reads; when none of the three is set, the format decides.
+ static const char FORMAT_TABLE_FILE_COMPRESSION[];
+
+ /// "format-table.partition-path-only-value" - Whether a format table names a partition
+ /// directory by its value alone ("2025/01/") instead of "key=value"
+ /// ("year=2025/month=01/"). Default false.
+ static const char FORMAT_TABLE_PARTITION_PATH_ONLY_VALUE[];
+
+ /// "metastore.partitioned-table" - Whether a table's partitions are registered with the
+ /// catalog, which then decides their visibility. Default false; paimon-cpp reads partitions
+ /// from the directory layout and rejects a table that sets it.
+ static const char METASTORE_PARTITIONED_TABLE[];
+
+ /// "file.suffix.include.compression" - Whether a data file's name carries the compression it
+ /// was written with, for a format that records it inside the file. Default false.
+ static const char FILE_SUFFIX_INCLUDE_COMPRESSION[];
+
/// "file-system" - Specify the file system.
/// Default value is local.
static const char FILE_SYSTEM[];
diff --git a/include/paimon/read_context.h b/include/paimon/read_context.h
index e645645b..8d34aab5 100644
--- a/include/paimon/read_context.h
+++ b/include/paimon/read_context.h
@@ -35,6 +35,7 @@
namespace paimon {
class Executor;
+class FormatTable;
class MemoryPool;
class Predicate;
class FileSystem;
@@ -60,7 +61,8 @@ class PAIMON_EXPORT ReadContext {
const std::map<std::string, std::string>& fs_scheme_to_identifier_map,
const std::shared_ptr<RealtimeContext>& realtime_context,
const std::map<std::string, std::string>& options, bool read_ahead_cache_enabled,
- const CacheConfig& cache_config, const std::shared_ptr<Cache>& cache);
+ const CacheConfig& cache_config, const std::shared_ptr<Cache>& cache,
+ const std::shared_ptr<FormatTable>& format_table);
~ReadContext();
const std::string& GetPath() const {
@@ -142,6 +144,12 @@ class PAIMON_EXPORT ReadContext {
return cache_;
}
+ /// The format table this context was built from, or null when it names a table path and the
+ /// schema under that path says what kind of table it is.
+ const std::shared_ptr<FormatTable>& GetFormatTable() const {
+ return format_table_;
+ }
+
/// Whether a read schema (C ArrowSchema) for nested column pruning was provided.
bool HasReadSchema() const {
return read_schema_ != nullptr && read_schema_->release != nullptr;
@@ -181,6 +189,7 @@ class PAIMON_EXPORT ReadContext {
bool read_ahead_cache_enabled_;
CacheConfig cache_config_;
std::shared_ptr<Cache> cache_;
+ std::shared_ptr<FormatTable> format_table_;
// Owns schema resources and releases ArrowSchema::release in destructor.
std::unique_ptr<ArrowSchema> read_schema_;
};
@@ -192,6 +201,15 @@ class PAIMON_EXPORT ReadContextBuilder {
/// @param path The root path of the table.
explicit ReadContextBuilder(const std::string& path);
+ /// Constructs a `ReadContextBuilder` for a format table that is already loaded: the only way
+ /// to read one whose schema lives in a metastore rather than under its location, such as a
+ /// table a REST catalog serves. The table carries what such a location does not say, so
+ /// `SetTableSchema()`, `WithFileSystem()`, `WithFileSystemSchemeToIdentifierMap()` and a
+ /// branch are refused here rather than ignored.
+ ///
+ /// @param table The format table to read, as `Catalog::GetFormatTable()` hands it back.
+ explicit ReadContextBuilder(const std::shared_ptr<FormatTable>& table);
+
~ReadContextBuilder();
ReadContextBuilder(ReadContextBuilder&&) noexcept;
diff --git a/include/paimon/scan_context.h b/include/paimon/scan_context.h
index 9c0b9d47..28906253 100644
--- a/include/paimon/scan_context.h
+++ b/include/paimon/scan_context.h
@@ -35,6 +35,7 @@ namespace paimon {
class ScanContextBuilder;
class ScanFilter;
class Executor;
+class FormatTable;
class MemoryPool;
class Predicate;
@@ -54,7 +55,8 @@ class PAIMON_EXPORT ScanContext {
const std::shared_ptr<FileSystem>& specific_file_system,
const std::optional<std::string>& table_schema,
const std::map<std::string, std::string>& options,
- const std::shared_ptr<Cache>& cache);
+ const std::shared_ptr<Cache>& cache,
+ const std::shared_ptr<FormatTable>& format_table);
~ScanContext();
@@ -105,6 +107,12 @@ class PAIMON_EXPORT ScanContext {
return cache_;
}
+ /// The format table this context was built from, or null when it names a table path and the
+ /// schema under that path says what kind of table it is.
+ const std::shared_ptr<FormatTable>& GetFormatTable() const {
+ return format_table_;
+ }
+
private:
std::string path_;
bool is_streaming_mode_;
@@ -118,6 +126,7 @@ class PAIMON_EXPORT ScanContext {
std::optional<std::string> table_schema_;
std::map<std::string, std::string> options_;
std::shared_ptr<Cache> cache_;
+ std::shared_ptr<FormatTable> format_table_;
};
/// Filter configuration for table scan operations
@@ -152,6 +161,15 @@ class PAIMON_EXPORT ScanContextBuilder {
/// Constructs a `ScanContextBuilder` with required parameters.
/// @param path The root path of the table.
explicit ScanContextBuilder(const std::string& path);
+
+ /// Constructs a `ScanContextBuilder` for a format table that is already loaded: the only way
+ /// to scan one whose schema lives in a metastore rather than under its location, such as a
+ /// table a REST catalog serves. The table carries what such a location does not say, so
+ /// `SetTableSchema()` and `WithFileSystem()` are refused here rather than ignored.
+ ///
+ /// @param table The format table to scan, as `Catalog::GetFormatTable()` hands it back.
+ explicit ScanContextBuilder(const std::shared_ptr<FormatTable>& table);
+
~ScanContextBuilder();
/// If limit is not set, it defaults to unlimited.
ScanContextBuilder& SetLimit(int32_t limit);
diff --git a/include/paimon/table/format/format_table.h b/include/paimon/table/format/format_table.h
new file mode 100644
index 00000000..3f07880e
--- /dev/null
+++ b/include/paimon/table/format/format_table.h
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "paimon/catalog/identifier.h"
+#include "paimon/result.h"
+#include "paimon/schema/schema.h"
+#include "paimon/status.h"
+#include "paimon/type_fwd.h"
+#include "paimon/visibility.h"
+
+struct ArrowSchema;
+
+namespace paimon {
+
+class FileSystem;
+
+/// A table that is a directory of data files of one format, laid out like a standard Hive table.
+///
+/// It carries no snapshots and no manifests: the files under its location are the table, and a
+/// partitioned table's partitions are the `key=value` directories below that location, or the
+/// bare-value ones under `format-table.partition-path-only-value`. A table is a format table when
+/// its `type` option is `format-table`; `file.format` then names the format of every file in it,
+/// defaulting to `parquet`.
+///
+/// Writes only insert - there is nowhere to record an update or a delete - so a read fills
+/// `_VALUE_KIND` with inserts throughout.
+///
+/// Reading and writing go through the entry points every other table uses: `TableScan`,
+/// `TableRead`, `FileStoreWrite` and `FileStoreCommit`. Each recognises a format table from the
+/// schema under the table path, or takes one already loaded through the `FormatTable` constructor
+/// every context builder has, which is how a table whose schema lives in a metastore is reached.
+///
+/// See `docs/source/user_guide/format_table.rst` for what is not supported yet.
+class PAIMON_EXPORT FormatTable {
+ public:
+ /// Formats a format table's files can be in.
+ enum class Format {
+ PARQUET,
+ ORC,
+ };
+
+ /// Parses the `file.format` option, case-insensitively. A format this library has no reader
+ /// for is rejected by name, instead of failing later with a missing-format-factory error.
+ static Result<Format> ParseFormat(const std::string& file_format);
+
+ /// The identifier of a format, as it appears in `file.format` and as a file extension.
+ static std::string FormatToString(Format format);
+
+ /// Loads a format table from its directory, reading the schema stored under it.
+ ///
+ /// This needs a schema file under the table directory, which a table created through
+ /// `SchemaManager` or a file system catalog has. A table whose schema lives in a metastore has
+ /// none, and is loaded through `Catalog::GetFormatTable()` instead.
+ ///
+ /// @param file_system File system holding the table directory.
+ /// @param table_path Root path of the table, which is also its data location.
+ /// @param identifier Logical table identifier, used for naming and error messages.
+ /// @param dynamic_options Options given at the call, which win over the ones stored in the
+ /// schema. Empty when the caller has none.
+ /// @return A result containing the format table, or an error status.
+ static Result<std::shared_ptr<FormatTable>> Create(
+ const std::shared_ptr<FileSystem>& file_system, const std::string& table_path,
+ const Identifier& identifier, const std::map<std::string, std::string>& dynamic_options);
+
+ /// Builds a format table from a schema that is already loaded, for a caller that has one in
+ /// hand, such as a catalog that just created the table.
+ ///
+ /// @param location Directory the data files live in. It may not be empty: every path this
+ /// table reads or writes is checked against it, and an empty one is a prefix of
+ /// nothing. A trailing separator names the same directory as none.
+ /// @param location_carries_paimon_metadata See `LocationCarriesPaimonMetadata()`. Only the
+ /// caller knows: a file system catalog puts metadata there, a REST or Hive catalog
+ /// keeps it in the metastore.
+ /// @param dynamic_options Options given at the call, which win over the ones stored in the
+ /// schema. Empty when the caller has none.
+ /// @return A result containing the format table, or an error status.
+ static Result<std::shared_ptr<FormatTable>> Create(
+ const std::shared_ptr<FileSystem>& file_system, const std::string& location,
+ const Identifier& identifier, const std::shared_ptr<DataSchema>& schema,
+ bool location_carries_paimon_metadata,
+ const std::map<std::string, std::string>& dynamic_options);
+
+ /// Copies `table` with `dynamic_options` on top of the options it already carries, which is
+ /// the precedence every context builder promises for a table it was handed rather than loaded
+ /// itself. `table` comes back as it is when there is nothing to add.
+ ///
+ /// `type` is not overridable: it is structural and is read from the schema alone.
+ ///
+ /// @param table The table to copy.
+ /// @param dynamic_options Options given at the call.
+ /// @return A result containing the copied table, or an error status.
+ static Result<std::shared_ptr<FormatTable>> Copy(
+ const std::shared_ptr<FormatTable>& table,
+ const std::map<std::string, std::string>& dynamic_options);
+
+ ~FormatTable();
+
+ /// Directory the data files live in.
+ const std::string& Location() const {
+ return location_;
+ }
+
+ /// Format of every data file in the directory.
+ Format GetFormat() const {
+ return format_;
+ }
+
+ /// Fields the table is partitioned by, in the order their directories nest.
+ const std::vector<std::string>& PartitionKeys() const;
+
+ /// Compression new data files are written with. It is resolved from `file.compression`, then
+ /// `format-table.file.compression`, then the bare `compression` key an engine's own writer
+ /// reads, then what the table's format writes by default.
+ const std::string& FileCompression() const {
+ return file_compression_;
+ }
+
+ /// Directory name standing for a null partition value, from `partition.default-name`.
+ const std::string& PartitionDefaultName() const {
+ return partition_default_name_;
+ }
+
+ /// Whether a partition directory is named by its value alone (`2025/01/`) instead of
+ /// `key=value` (`year=2025/month=01/`), from `format-table.partition-path-only-value`. The
+ /// value-only layout carries no field names, so the nesting order of the table's partition
+ /// keys alone says which key a directory belongs to.
+ bool PartitionOnlyValueInPath() const {
+ return partition_only_value_in_path_;
+ }
+
+ /// Table options: the ones stored in the schema, with any given at the call on top.
+ const std::map<std::string, std::string>& Options() const {
+ return options_;
+ }
+
+ /// A name to identify this table.
+ std::string Name() const {
+ return identifier_.GetTableName();
+ }
+
+ /// Full name of the table, database.tableName.
+ std::string FullName() const;
+
+ /// Schema of the table, including its partition fields.
+ std::shared_ptr<DataSchema> LatestSchema() const {
+ return schema_;
+ }
+
+ /// Schema of the table as an arrow schema, including its partition fields.
+ Result<std::unique_ptr<::ArrowSchema>> GetArrowSchema() const;
+
+ /// File system holding the table directory.
+ std::shared_ptr<FileSystem> GetFileSystem() const {
+ return file_system_;
+ }
+
+ /// Whether this table's own metadata lives under its location, as told by whoever loaded it.
+ ///
+ /// Only then are the `schema` and `branch` directories below the location table metadata
+ /// rather than table content. For a table whose schema lives in a metastore they are data,
+ /// and are read and written like any other directory.
+ bool LocationCarriesPaimonMetadata() const {
+ return location_carries_paimon_metadata_;
+ }
+
+ private:
+ FormatTable(const std::shared_ptr<FileSystem>& file_system, const std::string& location,
+ const Identifier& identifier, const std::shared_ptr<DataSchema>& schema,
+ const std::map<std::string, std::string>& options, Format format,
+ const std::string& file_compression, const std::string& partition_default_name,
+ bool partition_only_value_in_path, bool location_carries_paimon_metadata);
+
+ std::shared_ptr<FileSystem> file_system_;
+ std::string location_;
+ Identifier identifier_;
+ std::shared_ptr<DataSchema> schema_;
+ std::map<std::string, std::string> options_;
+ Format format_;
+ std::string file_compression_;
+ std::string partition_default_name_;
+ bool partition_only_value_in_path_ = false;
+ bool location_carries_paimon_metadata_ = false;
+};
+
+} // namespace paimon
diff --git a/include/paimon/table/source/split.h b/include/paimon/table/source/split.h
index a968f02c..7f296fb9 100644
--- a/include/paimon/table/source/split.h
+++ b/include/paimon/table/source/split.h
@@ -34,9 +34,12 @@ namespace paimon {
class MemoryPool;
/// An input split for reading operation. Needed by most batch computation engines. Support
-/// Serialize and Deserialize, compatible with java version.
-/// This split can be either a `DataSplit` (for direct data file reads) or an `IndexedSplit`
-/// (for reads leveraging global indexes).
+/// Serialize and Deserialize.
+///
+/// This split can be a `DataSplit` (for direct data file reads), an `IndexedSplit` (for reads
+/// leveraging global indexes), or a `FormatDataSplit` (for a format table's plain data files).
+/// Only the first two are serializable; a `FormatDataSplit` is in-memory only and `Serialize()`
+/// refuses it.
class PAIMON_EXPORT Split {
public:
virtual ~Split() = default;
diff --git a/include/paimon/table/source/table_scan.h b/include/paimon/table/source/table_scan.h
index 14c44756..8bae74c4 100644
--- a/include/paimon/table/source/table_scan.h
+++ b/include/paimon/table/source/table_scan.h
@@ -19,7 +19,10 @@
#pragma once
+#include <map>
#include <memory>
+#include <string>
+#include <vector>
#include "paimon/result.h"
#include "paimon/table/source/plan.h"
@@ -51,5 +54,15 @@ class PAIMON_EXPORT TableScan {
/// @return A point-in-time snapshot of scan metrics. Mutating the returned object does not
/// affect metrics collected by this scan.
virtual std::shared_ptr<Metrics> GetMetrics() const;
+
+ /// Lists the partitions the scan can see, each as its partition values keyed by field name,
+ /// in a stable order. A table with no partition keys has none.
+ ///
+ /// Only a format table answers it today: its partitions are the directories a scan descends,
+ /// so listing them costs no more than planning. Every other table type returns
+ /// `NotImplemented`.
+ ///
+ /// @return A Result containing the partitions, or an error status.
+ virtual Result<std::vector<std::map<std::string, std::string>>> ListPartitions() const;
};
} // namespace paimon
diff --git a/include/paimon/write_context.h b/include/paimon/write_context.h
index a9dd367e..16514d39 100644
--- a/include/paimon/write_context.h
+++ b/include/paimon/write_context.h
@@ -31,6 +31,7 @@
namespace paimon {
class Executor;
+class FormatTable;
class MemoryPool;
/// `WriteContext` is some configuration for write operations.
@@ -49,7 +50,8 @@ class PAIMON_EXPORT WriteContext {
const std::shared_ptr<FileSystem>& specific_file_system,
const std::map<std::string, std::string>& fs_scheme_to_identifier_map,
const std::shared_ptr<RealtimeContext>& realtime_context,
- const std::map<std::string, std::string>& options);
+ const std::map<std::string, std::string>& options,
+ const std::shared_ptr<FormatTable>& format_table);
~WriteContext();
@@ -118,6 +120,12 @@ class PAIMON_EXPORT WriteContext {
return realtime_context_;
}
+ /// The format table this context was built from, or null when it names a table path and the
+ /// schema under that path says what kind of table it is.
+ const std::shared_ptr<FormatTable>& GetFormatTable() const {
+ return format_table_;
+ }
+
private:
std::string root_path_;
std::string commit_user_;
@@ -135,6 +143,7 @@ class PAIMON_EXPORT WriteContext {
std::map<std::string, std::string> fs_scheme_to_identifier_map_;
std::shared_ptr<RealtimeContext> realtime_context_;
std::map<std::string, std::string> options_;
+ std::shared_ptr<FormatTable> format_table_;
};
/// `WriteContextBuilder` used to build a `WriteContext`, has input validation.
@@ -145,6 +154,17 @@ class PAIMON_EXPORT WriteContextBuilder {
/// @param commit_user The user identifier for commit operations.
WriteContextBuilder(const std::string& root_path, const std::string& commit_user);
+ /// Constructs a `WriteContextBuilder` for a format table that is already loaded: the only way
+ /// to write one whose schema lives in a metastore rather than under its location, such as a
+ /// table a REST catalog serves. The table carries what such a location does not say, so
+ /// `WithFileSystem()`, `WithFileSystemSchemeToIdentifierMap()` and a branch are refused here
+ /// rather than ignored.
+ ///
+ /// There is no commit user: a format table keeps no snapshot to record one in.
+ ///
+ /// @param table The format table to write to, as `Catalog::GetFormatTable()` hands it back.
+ explicit WriteContextBuilder(const std::shared_ptr<FormatTable>& table);
+
~WriteContextBuilder();
/// Set a configuration options map to set some option entries which are not defined in the
diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index 4c9b9b34..794c935b 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -136,6 +136,7 @@ set(PAIMON_COMMON_SRCS
common/reader/batch_reader.cpp
common/reader/concat_batch_reader.cpp
common/reader/late_materializing_file_batch_reader.cpp
+ common/reader/data_file_reader_factory.cpp
common/reader/predicate_batch_reader.cpp
common/reader/prefetch_file_batch_reader_impl.cpp
common/reader/reader_utils.cpp
@@ -409,6 +410,18 @@ set(PAIMON_CORE_SRCS
core/stats/simple_stats_evolution.cpp
core/table/table.cpp
core/table/bucket_mode.cpp
+ core/table/format/format_file_listing.cpp
+ core/table/format/format_file_naming.cpp
+ core/table/format/format_path_validation.cpp
+ core/table/format/format_table.cpp
+ core/table/format/format_table_commit.cpp
+ core/table/format/format_table_file_store_commit.cpp
+ core/table/format/format_table_file_store_write.cpp
+ core/table/format/format_table_loader.cpp
+ core/table/format/format_table_read.cpp
+ core/table/format/format_table_scan.cpp
+ core/table/format/format_table_write.cpp
+ core/table/format/lazy_concat_batch_reader.cpp
core/table/sink/commit_message.cpp
core/table/sink/commit_message_impl.cpp
core/table/sink/commit_message_serializer.cpp
@@ -634,6 +647,7 @@ if(PAIMON_BUILD_TESTS)
common/types/row_kind_test.cpp
common/types/data_type_test.cpp
common/utils/file_type_test.cpp
+ common/utils/hadoop_compression_test.cpp
common/utils/row_range_index_test.cpp
common/utils/var_length_int_utils_test.cpp
common/utils/arrow/arrow_utils_test.cpp
@@ -750,6 +764,7 @@ if(PAIMON_BUILD_TESTS)
core/compact/noop_compact_manager_test.cpp
core/core_options_test.cpp
core/options/lookup_strategy_test.cpp
+ core/options/table_type_test.cpp
core/deletionvectors/apply_deletion_vector_batch_reader_test.cpp
core/deletionvectors/bitmap_deletion_vector_test.cpp
core/deletionvectors/bucketed_dv_maintainer_test.cpp
@@ -909,6 +924,10 @@ if(PAIMON_BUILD_TESTS)
core/stats/simple_stats_test.cpp
core/table/table_test.cpp
core/table/bucket_mode_test.cpp
+ core/table/format/format_file_listing_test.cpp
+ core/table/format/format_file_naming_test.cpp
+ core/table/format/format_table_test.cpp
+ core/table/format/lazy_concat_batch_reader_test.cpp
core/table/sink/commit_message_test.cpp
core/table/sink/commit_message_impl_test.cpp
core/table/source/fallback_data_split_test.cpp
diff --git a/src/paimon/common/defs.cpp b/src/paimon/common/defs.cpp
index 962216f9..619ce848 100644
--- a/src/paimon/common/defs.cpp
+++ b/src/paimon/common/defs.cpp
@@ -36,7 +36,13 @@ const char Options::SEQUENCE_GROUP[] = "sequence-group";
const char Options::BUCKET[] = "bucket";
const char Options::BUCKET_KEY[] = "bucket-key";
+const char Options::TYPE[] = "type";
const char Options::FILE_FORMAT[] = "file.format";
+const char Options::FORMAT_TABLE_FILE_COMPRESSION[] = "format-table.file.compression";
+const char Options::FORMAT_TABLE_PARTITION_PATH_ONLY_VALUE[] =
+ "format-table.partition-path-only-value";
+const char Options::METASTORE_PARTITIONED_TABLE[] = "metastore.partitioned-table";
+const char Options::FILE_SUFFIX_INCLUDE_COMPRESSION[] = "file.suffix.include.compression";
const char Options::FILE_SYSTEM[] = "file-system";
const char Options::TARGET_FILE_SIZE[] = "target-file-size";
const char Options::TARGET_FILE_ROW_NUM[] = "target-file-row-num";
diff --git a/src/paimon/common/reader/data_file_reader_factory.cpp b/src/paimon/common/reader/data_file_reader_factory.cpp
new file mode 100644
index 00000000..4507127f
--- /dev/null
+++ b/src/paimon/common/reader/data_file_reader_factory.cpp
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/common/reader/data_file_reader_factory.h"
+
+#include <utility>
+
+#include "paimon/common/data/blob_defs.h"
+#include "paimon/common/reader/delegating_prefetch_reader.h"
+#include "paimon/common/reader/prefetch_file_batch_reader_impl.h"
+#include "paimon/format/file_format.h"
+#include "paimon/format/file_format_factory.h"
+#include "paimon/format/read_hints.h"
+#include "paimon/format/reader_builder.h"
+#include "paimon/fs/file_system.h"
+
+namespace paimon {
+
+namespace {
+
+/// Formats the prefetching reader cannot drive: `blob` is read whole rather than in batches and
+/// `avro` is row-oriented, so neither has the batch boundaries it reads ahead to. `mosaic` has
+/// never been tried under one.
+// TODO(xinyu.lxy): test mosaic under prefetch. A format table cannot hold mosaic files, so this
+// only concerns the managed table path.
+bool FormatSupportsPrefetch(const std::string& format_identifier) {
+ return format_identifier != "blob" && format_identifier != "avro" &&
+ format_identifier != "mosaic";
+}
+
+} // namespace
+
+Result<std::unique_ptr<ReaderBuilder>> DataFileReaderFactory::CreateReaderBuilder(
+ const std::string& format_identifier, const std::map<std::string, std::string>& format_options,
+ const std::map<std::string, std::string>& extra_format_options,
+ const DataFileReadOptions& read_options, const std::shared_ptr<MemoryPool>& pool) {
+ std::map<std::string, std::string> options = format_options;
+ // The blob placeholder channels are internal: a table option must not enable them, only
+ // `extra_format_options` may.
+ BlobDefs::EraseInternalPlaceholderOptions(&options);
+ for (const auto& [key, value] : extra_format_options) {
+ options[key] = value;
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileFormat> file_format,
+ FileFormatFactory::Get(format_identifier, options));
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<ReaderBuilder> reader_builder,
+ file_format->CreateReaderBuilder(read_options.read_batch_size));
+ reader_builder->WithMemoryPool(pool);
+ reader_builder->WithCache(read_options.cache);
+ // Runtime read state rather than more format options: parquet turns its own pre-buffering
+ // off when the shared read-ahead cache is reading ahead.
+ ReadHints read_hints;
+ read_hints.prefetch_enabled = read_options.prefetch_enabled;
+ read_hints.read_ahead_cache_enabled = read_options.read_ahead_cache_enabled;
+ reader_builder->WithReadHints(read_hints);
+ return reader_builder;
+}
+
+Result<std::unique_ptr<FileBatchReader>> DataFileReaderFactory::Open(
+ const std::string& format_identifier, const std::string& file_path, int64_t file_size,
+ const ReaderBuilder* reader_builder, const DataFileReadOptions& read_options,
+ const std::shared_ptr<FileSystem>& file_system, const std::shared_ptr<Executor>& executor,
+ const std::shared_ptr<MemoryPool>& pool, const std::shared_ptr<arrow::MemoryPool>& arrow_pool) {
+ if (read_options.prefetch_enabled && FormatSupportsPrefetch(format_identifier)) {
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<PrefetchFileBatchReaderImpl> prefetch_reader,
+ PrefetchFileBatchReaderImpl::Create(
+ file_path, file_size, reader_builder, file_system,
+ read_options.prefetch_max_parallel_num,
+ read_options.read_batch_size, read_options.prefetch_batch_count,
+ read_options.adaptive_prefetch_strategy, executor,
+ /*initialize_read_ranges=*/false,
+ read_options.read_ahead_cache_enabled, read_options.cache_config,
+ read_options.prefetch_io_metrics_enabled, pool, arrow_pool));
+ return std::make_unique<DelegatingPrefetchReader>(std::move(prefetch_reader));
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InputStream> input_stream,
+ file_system->Open(FileStatus(file_path, file_size)));
+ return reader_builder->Build(input_stream);
+}
+
+} // namespace paimon
diff --git a/src/paimon/common/reader/data_file_reader_factory.h b/src/paimon/common/reader/data_file_reader_factory.h
new file mode 100644
index 00000000..264ea34b
--- /dev/null
+++ b/src/paimon/common/reader/data_file_reader_factory.h
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <string>
+
+#include "paimon/read_context.h"
+#include "paimon/result.h"
+
+namespace arrow {
+class MemoryPool;
+} // namespace arrow
+
+namespace paimon {
+
+class Cache;
+class Executor;
+class FileBatchReader;
+class FileSystem;
+class MemoryPool;
+class ReaderBuilder;
+
+/// Settings applied when a data file is opened, gathered from the read context and the table
+/// options. The defaults are a plain read: no cache, no prefetch.
+struct DataFileReadOptions {
+ /// Block cache the format reader may put what it reads into, or null for none.
+ std::shared_ptr<Cache> cache;
+ int32_t read_batch_size = 0;
+ bool prefetch_enabled = false;
+ uint32_t prefetch_max_parallel_num = 0;
+ uint32_t prefetch_batch_count = 0;
+ bool adaptive_prefetch_strategy = false;
+ /// Whether the shared read-ahead cache takes over reading ahead from the format itself.
+ bool read_ahead_cache_enabled = true;
+ CacheConfig cache_config;
+ /// Whether the prefetching reader records what its I/O costs.
+ bool prefetch_io_metrics_enabled = false;
+};
+
+/// Opens one data file as a `FileBatchReader`.
+///
+/// Both table paths come through here - a managed table through `AbstractSplitRead`, a format
+/// table through `FormatTableRead` - so which format reads a file, what it may cache and whether
+/// it is read ahead are decided in one place rather than two that can drift.
+class DataFileReaderFactory {
+ public:
+ DataFileReaderFactory() = delete;
+ ~DataFileReaderFactory() = delete;
+
+ /// Builds the reader builder a format reads with. It is kept out of `Open()` so that a split
+ /// builds one and reuses it for every file.
+ ///
+ /// @param format_options The table's options, which the format reads its own settings from.
+ /// Internal blob placeholder options are dropped: only the internal read path may
+ /// enable those, through `extra_format_options`.
+ /// @param extra_format_options Options only the caller knows, applied over `format_options`.
+ static Result<std::unique_ptr<ReaderBuilder>> CreateReaderBuilder(
+ const std::string& format_identifier,
+ const std::map<std::string, std::string>& format_options,
+ const std::map<std::string, std::string>& extra_format_options,
+ const DataFileReadOptions& read_options, const std::shared_ptr<MemoryPool>& pool);
+
+ /// Opens `file_path`, reading ahead when the options ask for it and the format can.
+ ///
+ /// @param file_size Size of the file in bytes. It is trusted: an object-store read is issued
+ /// against it, so a caller holding a size it does not vouch for should check it against
+ /// the file system first.
+ static Result<std::unique_ptr<FileBatchReader>> Open(
+ const std::string& format_identifier, const std::string& file_path, int64_t file_size,
+ const ReaderBuilder* reader_builder, const DataFileReadOptions& read_options,
+ const std::shared_ptr<FileSystem>& file_system, const std::shared_ptr<Executor>& executor,
+ const std::shared_ptr<MemoryPool>& pool,
+ const std::shared_ptr<arrow::MemoryPool>& arrow_pool);
+};
+
+} // namespace paimon
diff --git a/src/paimon/common/utils/binary_row_partition_computer.cpp b/src/paimon/common/utils/binary_row_partition_computer.cpp
index ec773e32..d0be6f07 100644
--- a/src/paimon/common/utils/binary_row_partition_computer.cpp
+++ b/src/paimon/common/utils/binary_row_partition_computer.cpp
@@ -115,7 +115,7 @@ Result<BinaryRow> BinaryRowPartitionComputer::ConvertToBinaryRow(
return binary_row;
}
-Result<std::map<std::string, std::string>> BinaryRowPartitionComputer::NormalizePartitionSpec(
+Status BinaryRowPartitionComputer::CheckPartitionKeys(
const std::map<std::string, std::string>& partition) const {
for (const auto& [partition_key, _] : partition) {
if (std::find(partition_keys_.begin(), partition_keys_.end(), partition_key) ==
@@ -124,6 +124,19 @@ Result<std::map<std::string, std::string>> BinaryRowPartitionComputer::Normalize
fmt::format("field {} does not exist in partition keys", partition_key));
}
}
+ return Status::OK();
+}
+
+Result<BinaryRow> BinaryRowPartitionComputer::ToPartialBinaryRow(
+ const std::map<std::string, std::string>& partition) const {
+ PAIMON_RETURN_NOT_OK(CheckPartitionKeys(partition));
+ std::vector<bool> included_fields;
+ return ConvertToBinaryRow(partition, &included_fields);
+}
+
+Result<std::map<std::string, std::string>> BinaryRowPartitionComputer::NormalizePartitionSpec(
+ const std::map<std::string, std::string>& partition) const {
+ PAIMON_RETURN_NOT_OK(CheckPartitionKeys(partition));
std::vector<bool> included_fields;
PAIMON_ASSIGN_OR_RAISE(BinaryRow binary_row, ConvertToBinaryRow(partition, &included_fields));
diff --git a/src/paimon/common/utils/binary_row_partition_computer.h b/src/paimon/common/utils/binary_row_partition_computer.h
index 2e96ad6f..08990d61 100644
--- a/src/paimon/common/utils/binary_row_partition_computer.h
+++ b/src/paimon/common/utils/binary_row_partition_computer.h
@@ -55,6 +55,17 @@ class BinaryRowPartitionComputer {
bool legacy_partition_name_enabled, const std::shared_ptr<MemoryPool>& memory_pool);
Result<BinaryRow> ToBinaryRow(const std::map<std::string, std::string>& partition) const;
+
+ /// Converts a partial `partition` into a `BinaryRow`: a key it names is read into that
+ /// column's type, a key it leaves out is null, and a value equal to the default partition name
+ /// is the null partition.
+ ///
+ /// Nothing is rendered back out, unlike `NormalizePartitionSpec()`, so two values a partition
+ /// directory's name cannot tell apart - a blank one and the null partition - stay distinct.
+ /// This is how a partition is compared by what its column holds, which is what Java tests a
+ /// `PartitionPredicate` against.
+ Result<BinaryRow> ToPartialBinaryRow(const std::map<std::string, std::string>& partition) const;
+
Result<std::map<std::string, std::string>> NormalizePartitionSpec(
const std::map<std::string, std::string>& partition) const;
Result<std::vector<std::pair<std::string, std::string>>> GeneratePartitionVector(
@@ -79,6 +90,10 @@ class BinaryRowPartitionComputer {
Result<BinaryRow> ConvertToBinaryRow(const std::map<std::string, std::string>& partition,
std::vector<bool>* included_fields) const;
+ /// Rejects a key that is no partition key of this table, so a typo matches nothing rather than
+ /// being ignored.
+ Status CheckPartitionKeys(const std::map<std::string, std::string>& partition) const;
+
static Result<arrow::Type::type> GetTypeFromArrowSchema(
const std::shared_ptr<arrow::Schema>& schema, const std::string& field_name);
diff --git a/src/paimon/common/utils/binary_row_partition_computer_test.cpp b/src/paimon/common/utils/binary_row_partition_computer_test.cpp
index 2a9d6b2d..4f60781e 100644
--- a/src/paimon/common/utils/binary_row_partition_computer_test.cpp
+++ b/src/paimon/common/utils/binary_row_partition_computer_test.cpp
@@ -305,6 +305,44 @@ TEST(BinaryRowPartitionComputerTest, TestNormalizePartialPartitionSpec) {
"field unknown does not exist in partition keys");
}
+TEST(BinaryRowPartitionComputerTest, TestToPartialBinaryRow) {
+ std::shared_ptr<MemoryPool> pool = GetDefaultPool();
+ std::shared_ptr<arrow::Schema> schema =
+ arrow::schema({arrow::field("pt", arrow::int32()), arrow::field("region", arrow::utf8())});
+ std::vector<std::string> partition_keys = {"pt", "region"};
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<BinaryRowPartitionComputer> computer,
+ BinaryRowPartitionComputer::Create(partition_keys, schema, "__DEFAULT_PARTITION__",
+ /*legacy_partition_name_enabled=*/true, pool));
+
+ // A key the spec leaves out is null, and two spellings of the same value give the same row,
+ // which is what makes the row comparable.
+ ASSERT_OK_AND_ASSIGN(BinaryRow one, computer->ToPartialBinaryRow({{"pt", "1"}}));
+ ASSERT_OK_AND_ASSIGN(BinaryRow padded_one, computer->ToPartialBinaryRow({{"pt", "01"}}));
+ ASSERT_EQ(one, padded_one);
+ ASSERT_FALSE(one.IsNullAt(0));
+ ASSERT_TRUE(one.IsNullAt(1));
+
+ // The default partition name is the null partition; a blank value is not, unlike what
+ // `NormalizePartitionSpec()` renders back out.
+ ASSERT_OK_AND_ASSIGN(BinaryRow null_region,
+ computer->ToPartialBinaryRow({{"region", "__DEFAULT_PARTITION__"}}));
+ ASSERT_OK_AND_ASSIGN(BinaryRow blank_region, computer->ToPartialBinaryRow({{"region", " "}}));
+ ASSERT_TRUE(null_region.IsNullAt(1));
+ ASSERT_FALSE(blank_region.IsNullAt(1));
+ ASSERT_FALSE(null_region == blank_region);
+ using PartitionSpec = std::map<std::string, std::string>;
+ ASSERT_OK_AND_ASSIGN(PartitionSpec null_normalized,
+ computer->NormalizePartitionSpec({{"region", "__DEFAULT_PARTITION__"}}));
+ ASSERT_OK_AND_ASSIGN(PartitionSpec blank_normalized,
+ computer->NormalizePartitionSpec({{"region", " "}}));
+ ASSERT_EQ(null_normalized, blank_normalized);
+
+ ASSERT_NOK_WITH_MSG(computer->ToPartialBinaryRow({{"unknown", "value"}}),
+ "field unknown does not exist in partition keys");
+ ASSERT_NOK(computer->ToPartialBinaryRow({{"pt", "abc"}}));
+}
+
TEST(BinaryRowPartitionComputerTest, TestPartToSimpleString) {
auto pool = GetDefaultPool();
{
diff --git a/src/paimon/common/utils/hadoop_compression.h b/src/paimon/common/utils/hadoop_compression.h
new file mode 100644
index 00000000..38a7516a
--- /dev/null
+++ b/src/paimon/common/utils/hadoop_compression.h
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <array>
+#include <optional>
+#include <string>
+
+#include "paimon/common/utils/string_utils.h"
+
+namespace paimon {
+
+/// The compressions hadoop names in a file's extension.
+///
+/// A data file's name carries its compression when it is one of these -
+/// `data-<uuid>-0.snappy.parquet` - and the option's own text otherwise.
+///
+/// Every name and extension below is fixed by what the rest of the paimon ecosystem writes, not
+/// chosen here: `zstd` is spelled `zst` in a file name because that is the extension those files
+/// carry. Changing one would not stop a parquet or orc file being read - both record their
+/// compression inside - but it would break the naming convention shared with Java Paimon.
+class HadoopCompression {
+ public:
+ enum class Kind {
+ NONE,
+ GZIP,
+ BZIP2,
+ DEFLATE,
+ SNAPPY,
+ LZ4,
+ ZSTD,
+ };
+
+ HadoopCompression() = delete;
+ ~HadoopCompression() = delete;
+
+ /// The compression a `file.compression` value names, case-insensitively. An empty value and
+ /// "none" are `NONE`; a value naming no compression at all is nullopt, and a caller writes
+ /// that one into the file name verbatim. Only the names listed here are recognised, so
+ /// "uncompressed" is not one of them.
+ static std::optional<Kind> FromName(const std::string& name) {
+ std::string normalized = StringUtils::ToLowerCase(name);
+ if (normalized.empty() || normalized == "none") {
+ return Kind::NONE;
+ }
+ for (const Kind kind : AllCompressions()) {
+ if (normalized == ToName(kind)) {
+ return kind;
+ }
+ }
+ return std::nullopt;
+ }
+
+ /// The extension the compression adds to a file name, without its dot. Empty for `NONE`.
+ static std::string ToFileExtension(Kind kind) {
+ switch (kind) {
+ case Kind::GZIP:
+ return "gz";
+ case Kind::BZIP2:
+ return "bz2";
+ case Kind::DEFLATE:
+ return "deflate";
+ case Kind::SNAPPY:
+ return "snappy";
+ case Kind::LZ4:
+ return "lz4";
+ case Kind::ZSTD:
+ return "zst";
+ case Kind::NONE:
+ break;
+ }
+ return std::string();
+ }
+
+ private:
+ /// The name a `file.compression` value carries for the compression.
+ static std::string ToName(Kind kind) {
+ switch (kind) {
+ case Kind::GZIP:
+ return "gzip";
+ case Kind::BZIP2:
+ return "bzip2";
+ case Kind::DEFLATE:
+ return "deflate";
+ case Kind::SNAPPY:
+ return "snappy";
+ case Kind::LZ4:
+ return "lz4";
+ case Kind::ZSTD:
+ return "zstd";
+ case Kind::NONE:
+ break;
+ }
+ return std::string();
+ }
+
+ static const std::array<Kind, 6>& AllCompressions() {
+ static const std::array<Kind, 6> kAll = {Kind::GZIP, Kind::BZIP2, Kind::DEFLATE,
+ Kind::SNAPPY, Kind::LZ4, Kind::ZSTD};
+ return kAll;
+ }
+};
+
+} // namespace paimon
diff --git a/src/paimon/common/utils/hadoop_compression_test.cpp b/src/paimon/common/utils/hadoop_compression_test.cpp
new file mode 100644
index 00000000..c25c6df2
--- /dev/null
+++ b/src/paimon/common/utils/hadoop_compression_test.cpp
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/common/utils/hadoop_compression.h"
+
+#include <optional>
+#include <string>
+
+#include "gtest/gtest.h"
+
+namespace paimon::test {
+
+TEST(HadoopCompressionTest, TestEveryNameMapsToItsFileExtension) {
+ // Each pair is fixed by what the rest of the paimon ecosystem writes and reads, not chosen
+ // here: getting one wrong makes files written here unreadable elsewhere, and silently, since
+ // the name is only ever appended to a file name. `zstd` spelled `zst` is the one that reads
+ // like a typo and is not.
+ struct Case {
+ const char* name;
+ HadoopCompression::Kind kind;
+ const char* extension;
+ };
+ for (const Case& test_case : {Case{"gzip", HadoopCompression::Kind::GZIP, "gz"},
+ Case{"bzip2", HadoopCompression::Kind::BZIP2, "bz2"},
+ Case{"deflate", HadoopCompression::Kind::DEFLATE, "deflate"},
+ Case{"snappy", HadoopCompression::Kind::SNAPPY, "snappy"},
+ Case{"lz4", HadoopCompression::Kind::LZ4, "lz4"},
+ Case{"zstd", HadoopCompression::Kind::ZSTD, "zst"}}) {
+ SCOPED_TRACE(test_case.name);
+ std::optional<HadoopCompression::Kind> kind = HadoopCompression::FromName(test_case.name);
+ ASSERT_TRUE(kind.has_value());
+ ASSERT_EQ(*kind, test_case.kind);
+ ASSERT_EQ(HadoopCompression::ToFileExtension(test_case.kind), test_case.extension);
+ }
+}
+
+TEST(HadoopCompressionTest, TestNoCompressionAddsNoExtension) {
+ // An empty value and "none" both mean no compression, which adds nothing to a file name.
+ for (const char* name : {"", "none", "NONE", "None"}) {
+ SCOPED_TRACE(name);
+ std::optional<HadoopCompression::Kind> kind = HadoopCompression::FromName(name);
+ ASSERT_TRUE(kind.has_value());
+ ASSERT_EQ(*kind, HadoopCompression::Kind::NONE);
+ }
+ ASSERT_TRUE(HadoopCompression::ToFileExtension(HadoopCompression::Kind::NONE).empty());
+}
+
+TEST(HadoopCompressionTest, TestANameIsMatchedIgnoringCase) {
+ for (const char* name : {"ZSTD", "Zstd", "zStD"}) {
+ SCOPED_TRACE(name);
+ std::optional<HadoopCompression::Kind> kind = HadoopCompression::FromName(name);
+ ASSERT_TRUE(kind.has_value());
+ ASSERT_EQ(*kind, HadoopCompression::Kind::ZSTD);
+ }
+}
+
+TEST(HadoopCompressionTest, TestAValueNamingNoCompressionIsNotOne) {
+ // A caller writes such a value into the file name verbatim, so it must come back as nullopt
+ // rather than as a compression whose extension would replace it. "uncompressed" is the one
+ // that reads like it belongs here: hadoop does not name it, so it does not.
+ for (const char* name : {"uncompressed", "zst", "gz", "brotli", "nonesuch"}) {
+ SCOPED_TRACE(name);
+ ASSERT_FALSE(HadoopCompression::FromName(name).has_value());
+ }
+}
+
+} // namespace paimon::test
diff --git a/src/paimon/common/utils/string_utils.cpp b/src/paimon/common/utils/string_utils.cpp
index 7b3ed96b..2e356515 100644
--- a/src/paimon/common/utils/string_utils.cpp
+++ b/src/paimon/common/utils/string_utils.cpp
@@ -137,7 +137,7 @@ bool StringUtils::IsBlank(std::string_view str) {
return true;
}
-bool StringUtils::IsNullOrWhitespaceOnly(const std::string& str) {
+bool StringUtils::IsNullOrWhitespaceOnly(std::string_view str) {
return IsBlank(str);
}
diff --git a/src/paimon/common/utils/string_utils.h b/src/paimon/common/utils/string_utils.h
index 49c0aff2..91f1427c 100644
--- a/src/paimon/common/utils/string_utils.h
+++ b/src/paimon/common/utils/string_utils.h
@@ -101,7 +101,9 @@ class PAIMON_EXPORT StringUtils {
/// Character.isWhitespace.
static bool IsBlank(std::string_view str);
- static bool IsNullOrWhitespaceOnly(const std::string& str);
+ /// Whether `str` is empty or holds nothing but whitespace. Takes a view so that a caller
+ /// checking one row of a column at a time does not allocate.
+ static bool IsNullOrWhitespaceOnly(std::string_view str);
/// Returns true if Java String::trim would produce an empty string.
static bool IsEmptyAfterTrim(std::string_view str);
diff --git a/src/paimon/core/catalog/catalog.cpp b/src/paimon/core/catalog/catalog.cpp
index 97f13afa..805cc972 100644
--- a/src/paimon/core/catalog/catalog.cpp
+++ b/src/paimon/core/catalog/catalog.cpp
@@ -20,10 +20,14 @@
#include <utility>
+#include "fmt/format.h"
#include "paimon/catalog_options.h"
#include "paimon/common/utils/string_utils.h"
+#include "paimon/core/catalog/catalog_utils.h"
#include "paimon/core/catalog/file_system_catalog.h"
#include "paimon/core/core_options.h"
+#include "paimon/schema/schema.h"
+#include "paimon/table/format/format_table.h"
#ifdef PAIMON_ENABLE_REST
#include "paimon/rest/rest_catalog.h"
#endif
@@ -60,4 +64,19 @@ Result<std::unique_ptr<Catalog>> Catalog::Create(const std::string& root_path,
return std::make_unique<FileSystemCatalog>(core_options.GetFileSystem(), root_path, options);
}
+Result<std::shared_ptr<FormatTable>> Catalog::GetFormatTable(const Identifier& identifier) const {
+ // Refused here rather than in each override, so every catalog rejects a system table the same
+ // way and in the same order: the database first, so that an identifier of "sys" is rejected
+ // without being parsed as a table name.
+ PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(identifier, "getFormatTable"));
+ return LoadFormatTable(identifier);
+}
+
+Result<std::shared_ptr<FormatTable>> Catalog::LoadFormatTable(const Identifier& identifier) const {
+ // The location and the schema are read separately, and every directory below the location is
+ // table content: a catalog that knows better overrides this and says so.
+ return CatalogUtils::LoadFormatTableInTwoRequests(*this, identifier,
+ /*location_carries_paimon_metadata=*/false);
+}
+
} // namespace paimon
diff --git a/src/paimon/core/catalog/catalog_utils.cpp b/src/paimon/core/catalog/catalog_utils.cpp
index 104a71be..72f817dc 100644
--- a/src/paimon/core/catalog/catalog_utils.cpp
+++ b/src/paimon/core/catalog/catalog_utils.cpp
@@ -18,13 +18,20 @@
#include "paimon/core/catalog/catalog_utils.h"
+#include <map>
+#include <memory>
#include <optional>
+#include <string>
#include "fmt/format.h"
#include "paimon/catalog/catalog.h"
#include "paimon/common/utils/path_util.h"
+#include "paimon/core/options/table_type.h"
#include "paimon/core/utils/branch_manager.h"
+#include "paimon/defs.h"
#include "paimon/result.h"
+#include "paimon/schema/schema.h"
+#include "paimon/table/format/format_table.h"
namespace paimon {
@@ -94,4 +101,46 @@ Status CatalogUtils::CheckValidTableName(const Identifier& identifier) {
return Status::OK();
}
+Status CatalogUtils::CheckManagedTableType(const Identifier& identifier,
+ const std::shared_ptr<Schema>& schema,
+ const std::string& action) {
+ std::shared_ptr<DataSchema> data_schema = std::dynamic_pointer_cast<DataSchema>(schema);
+ if (data_schema == nullptr) {
+ // Only a data table carries table options, so nothing else declares a table type.
+ return Status::OK();
+ }
+ PAIMON_ASSIGN_OR_RAISE(TableType table_type,
+ TableTypeDefine::FromOptions(data_schema->Options()));
+ if (table_type == TableType::FORMAT_TABLE) {
+ return Status::Invalid(
+ fmt::format("Cannot open format table '{}' as a Table in '{}', please use "
+ "'Catalog::GetFormatTable' or 'FormatTable::Create'.",
+ identifier.ToString(), action));
+ }
+ // A materialized table is a managed table that also carries the SQL it materializes.
+ if (table_type != TableType::TABLE && table_type != TableType::MATERIALIZED_TABLE) {
+ const std::map<std::string, std::string>& options = data_schema->Options();
+ auto type_iter = options.find(Options::TYPE);
+ return Status::NotImplemented(fmt::format(
+ "Cannot open table '{}' in '{}': its '{}' is '{}', a table type paimon-cpp does not "
+ "implement, and a managed table would promise snapshots it never had.",
+ identifier.ToString(), action, Options::TYPE,
+ type_iter == options.end() ? std::string() : type_iter->second));
+ }
+ return Status::OK();
+}
+
+Result<std::shared_ptr<FormatTable>> CatalogUtils::LoadFormatTableInTwoRequests(
+ const Catalog& catalog, const Identifier& identifier, bool location_carries_paimon_metadata) {
+ PAIMON_ASSIGN_OR_RAISE(std::string location, catalog.GetTableLocation(identifier));
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Schema> schema, catalog.LoadTableSchema(identifier));
+ std::shared_ptr<DataSchema> data_schema = std::dynamic_pointer_cast<DataSchema>(schema);
+ if (data_schema == nullptr) {
+ return Status::Invalid(fmt::format("{} is not a data table, so it cannot be a format table",
+ identifier.GetFullName()));
+ }
+ return FormatTable::Create(catalog.GetFileSystem(), location, identifier, data_schema,
+ location_carries_paimon_metadata, /*dynamic_options=*/{});
+}
+
} // namespace paimon
diff --git a/src/paimon/core/catalog/catalog_utils.h b/src/paimon/core/catalog/catalog_utils.h
index 235ad19d..4c0ea6d2 100644
--- a/src/paimon/core/catalog/catalog_utils.h
+++ b/src/paimon/core/catalog/catalog_utils.h
@@ -18,13 +18,19 @@
#pragma once
+#include <memory>
#include <string>
#include "paimon/catalog/identifier.h"
+#include "paimon/result.h"
#include "paimon/status.h"
namespace paimon {
+class Catalog;
+class FormatTable;
+class Schema;
+
/// Checks shared by the catalog implementations. Every check takes an `action` naming
/// the rejected operation in the error message, e.g. "dropTable".
class CatalogUtils {
@@ -51,6 +57,33 @@ class CatalogUtils {
/// Fails when any component parsed out of the identifier's table name (data table name,
/// branch name, system table name) cannot be used as a single path component.
static Status CheckValidTableName(const Identifier& identifier);
+
+ /// Fails when `schema` is not a table a `Table` can describe.
+ ///
+ /// Only a managed table is one: a format table is loaded through `GetFormatTable()`, and the
+ /// remaining table types are stored differently and are not implemented here, so handing one
+ /// back as a `Table` would promise snapshots it never had. A schema that is not a data
+ /// table's, such as a system table's, passes.
+ ///
+ /// @param action The call being refused, qualified as a caller would write it, e.g.
+ /// `Catalog::GetTable`. It names the entry point in the message.
+ static Status CheckManagedTableType(const Identifier& identifier,
+ const std::shared_ptr<Schema>& schema,
+ const std::string& action);
+
+ /// Loads `identifier` as a format table by reading its location and its schema separately.
+ ///
+ /// What every catalog can do, so it is what `Catalog::GetFormatTable()` falls back to. A
+ /// catalog that can get both from one response should not use it, since two requests can
+ /// disagree.
+ ///
+ /// @param location_carries_paimon_metadata Whether this catalog put the table's own metadata
+ /// under the table path, which is what tells a `schema` or `branch` directory below the
+ /// location from a partition directory of the same name. See
+ /// `FormatTable::LocationCarriesPaimonMetadata()`.
+ static Result<std::shared_ptr<FormatTable>> LoadFormatTableInTwoRequests(
+ const Catalog& catalog, const Identifier& identifier,
+ bool location_carries_paimon_metadata);
};
} // namespace paimon
diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp
index be5f11bb..742a490b 100644
--- a/src/paimon/core/catalog/file_system_catalog.cpp
+++ b/src/paimon/core/catalog/file_system_catalog.cpp
@@ -337,6 +337,7 @@ Result<std::shared_ptr<Table>> FileSystemCatalog::GetTable(const Identifier& ide
identifier.GetTableName());
}
PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(identifier));
+ // `Table::Create` reads the same schema file and rejects a format table there.
return Table::Create(fs_, table_path, identifier);
}
@@ -540,4 +541,10 @@ Result<std::vector<SnapshotInfo>> FileSystemCatalog::ListSnapshots(
return result;
}
+Result<std::shared_ptr<FormatTable>> FileSystemCatalog::LoadFormatTable(
+ const Identifier& identifier) const {
+ return CatalogUtils::LoadFormatTableInTwoRequests(*this, identifier,
+ /*location_carries_paimon_metadata=*/true);
+}
+
} // namespace paimon
diff --git a/src/paimon/core/catalog/file_system_catalog.h b/src/paimon/core/catalog/file_system_catalog.h
index a373dcfa..2b32bdb4 100644
--- a/src/paimon/core/catalog/file_system_catalog.h
+++ b/src/paimon/core/catalog/file_system_catalog.h
@@ -69,6 +69,13 @@ class FileSystemCatalog : public Catalog {
Result<std::vector<SnapshotInfo>> ListSnapshots(const Identifier& identifier,
const std::string& branch) const override;
+ protected:
+ /// This catalog keeps a table's schema under the table's own path, so the `schema` and
+ /// `branch` directories there are metadata rather than table content. Nothing else differs,
+ /// so the location and the schema are still read separately.
+ Result<std::shared_ptr<FormatTable>> LoadFormatTable(
+ const Identifier& identifier) const override;
+
private:
/// Fails when `db_name` cannot be used as a single path component, so that the returned
/// path always stays under `warehouse`.
diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp
index dba3e67c..bdfcf92d 100644
--- a/src/paimon/core/catalog/file_system_catalog_test.cpp
+++ b/src/paimon/core/catalog/file_system_catalog_test.cpp
@@ -36,6 +36,7 @@
#include "paimon/fs/file_system.h"
#include "paimon/fs/file_system_factory.h"
#include "paimon/snapshot/snapshot_info.h"
+#include "paimon/table/format/format_table.h"
#include "paimon/testing/utils/testharness.h"
namespace paimon::test {
@@ -530,6 +531,50 @@ TEST(FileSystemCatalogTest, TestCreateTableWithBlob) {
ArrowSchemaRelease(&schema);
}
+TEST(FileSystemCatalogTest, TestGetTableRejectsFormatTable) {
+ std::map<std::string, std::string> options;
+ options[Options::FILE_SYSTEM] = "local";
+ options[Options::FILE_FORMAT] = "orc";
+ ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options));
+ auto dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options);
+ ASSERT_OK(catalog.CreateDatabase("db1", options, /*ignore_if_exists=*/false));
+
+ arrow::Schema typed_schema(
+ {arrow::field("id", arrow::int32()), arrow::field("name", arrow::utf8())});
+ ::ArrowSchema schema;
+ ASSERT_TRUE(arrow::ExportSchema(typed_schema, &schema).ok());
+ std::map<std::string, std::string> table_options = {{Options::TYPE, "format-table"},
+ {Options::FILE_FORMAT, "parquet"}};
+ ASSERT_OK(catalog.CreateTable(Identifier("db1", "fmt"), &schema, /*partition_keys=*/{},
+ /*primary_keys=*/{}, table_options,
+ /*ignore_if_exists=*/false));
+ ArrowSchemaRelease(&schema);
+
+ // A format table has no snapshots and no manifests, so handing it back as a Table would
+ // describe it as something it is not.
+ ASSERT_NOK_WITH_MSG(catalog.GetTable(Identifier("db1", "fmt")), "Cannot open format table");
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> format_table,
+ catalog.GetFormatTable(Identifier("db1", "fmt")));
+ ASSERT_EQ(format_table->GetFormat(), FormatTable::Format::PARQUET);
+ ASSERT_EQ(format_table->FullName(), "db1.fmt");
+
+ // This catalog keeps a table's schema under the table's own path, so the `schema` and
+ // `branch` directories there are its metadata and a scan leaves them alone. A catalog that
+ // keeps schemas elsewhere says no, and then every directory below the location is data.
+ ASSERT_TRUE(format_table->LocationCarriesPaimonMetadata());
+
+ // A system table is refused by `Catalog::GetFormatTable()` itself, so every catalog answers
+ // alike. The database is checked before the table name is parsed, so a malformed name under
+ // "sys" is refused as a system table rather than surfacing a parse error.
+ ASSERT_NOK_WITH_MSG(catalog.GetFormatTable(Identifier("db1", "fmt$snapshots")),
+ "for system table");
+ ASSERT_NOK_WITH_MSG(catalog.GetFormatTable(Identifier("sys", "all_table_options")),
+ "for system table");
+ ASSERT_NOK_WITH_MSG(catalog.GetFormatTable(Identifier("sys", "a$b$c$d")), "for system table");
+}
+
TEST(FileSystemCatalogTest, TestInvalidCreateTable) {
std::map<std::string, std::string> options;
options[Options::FILE_SYSTEM] = "local";
diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp
index aaa5b645..ffb4fffe 100644
--- a/src/paimon/core/core_options.cpp
+++ b/src/paimon/core/core_options.cpp
@@ -328,6 +328,10 @@ class ConfigParser {
// Impl is a private implementation of CoreOptions,
// storing various configurable fields and their default values.
+// The fields are grouped by the option each one parses, so that a new option lands beside the code
+// that reads it. Reordering them by size would save a few dozen bytes in the one instance a table
+// holds, at the cost of that grouping.
+// NOLINTNEXTLINE(clang-analyzer-optin.performance.Padding)
struct CoreOptions::Impl {
int64_t page_size = 64 * 1024;
int64_t target_file_row_num = std::numeric_limits<int64_t>::max();
@@ -457,6 +461,9 @@ struct CoreOptions::Impl {
bool blob_view_resolve_enabled = true;
bool blob_as_descriptor = false;
bool legacy_partition_name_enabled = true;
+ bool file_suffix_include_compression = false;
+ bool format_table_partition_only_value_in_path = false;
+ bool metastore_partitioned_table = false;
bool global_index_enabled = true;
bool commit_force_compact = false;
bool commit_discard_duplicate_files = false;
@@ -605,6 +612,15 @@ struct CoreOptions::Impl {
PAIMON_RETURN_NOT_OK(parser.ParseFileFormatPerLevel(&file_format_per_level));
// Parse file.compression.per.level - different compression for different levels
PAIMON_RETURN_NOT_OK(parser.ParseFileCompressionPerLevel(&file_compression_per_level));
+ // Parse file.suffix.include.compression - carry the compression in a data file's name
+ PAIMON_RETURN_NOT_OK(parser.Parse<bool>(Options::FILE_SUFFIX_INCLUDE_COMPRESSION,
+ &file_suffix_include_compression));
+ // Parse format-table.partition-path-only-value - name a partition directory by its value
+ PAIMON_RETURN_NOT_OK(parser.Parse<bool>(Options::FORMAT_TABLE_PARTITION_PATH_ONLY_VALUE,
+ &format_table_partition_only_value_in_path));
+ // Parse metastore.partitioned-table - partitions are registered with the catalog
+ PAIMON_RETURN_NOT_OK(
+ parser.Parse<bool>(Options::METASTORE_PARTITIONED_TABLE, &metastore_partitioned_table));
return Status::OK();
}
@@ -1760,6 +1776,36 @@ bool CoreOptions::LegacyPartitionNameEnabled() const {
return impl_->legacy_partition_name_enabled;
}
+bool CoreOptions::FileSuffixIncludeCompression() const {
+ return impl_->file_suffix_include_compression;
+}
+
+bool CoreOptions::FormatTablePartitionOnlyValueInPath() const {
+ return impl_->format_table_partition_only_value_in_path;
+}
+
+bool CoreOptions::MetastorePartitionedTable() const {
+ return impl_->metastore_partitioned_table;
+}
+
+std::string CoreOptions::FormatTableFileCompression() const {
+ // The resolution order the rest of the paimon ecosystem follows; both the compression suffix
+ // in a file's name and its contents derive from it, so they cannot disagree. `compression` is
+ // not a paimon option of its own but the key an engine's own writer reads: paimon-spark copies
+ // `format-table.file.compression` onto it, so a table written that way carries only that key.
+ const char* const keys[] = {Options::FILE_COMPRESSION, Options::FORMAT_TABLE_FILE_COMPRESSION,
+ "compression"};
+ for (const char* key : keys) {
+ auto iter = impl_->raw_options.find(key);
+ if (iter != impl_->raw_options.end()) {
+ return iter->second;
+ }
+ }
+ // What the format writes by default, when no option names a compression.
+ return impl_->file_format != nullptr && impl_->file_format->Identifier() == "parquet" ? "snappy"
+ : "zstd";
+}
+
bool CoreOptions::GlobalIndexEnabled() const {
return impl_->global_index_enabled;
}
diff --git a/src/paimon/core/core_options.h b/src/paimon/core/core_options.h
index 345958e1..5c2c5e45 100644
--- a/src/paimon/core/core_options.h
+++ b/src/paimon/core/core_options.h
@@ -245,6 +245,23 @@ class PAIMON_EXPORT CoreOptions {
bool LegacyPartitionNameEnabled() const;
+ /// Whether a data file's name carries the compression it was written with, from
+ /// `file.suffix.include.compression`.
+ bool FileSuffixIncludeCompression() const;
+
+ /// Whether a format table names a partition directory by its value alone (`2025/01/`) instead
+ /// of `key=value` (`year=2025/month=01/`), from `format-table.partition-path-only-value`.
+ bool FormatTablePartitionOnlyValueInPath() const;
+
+ /// Whether the table's partitions are registered with the catalog rather than discovered from
+ /// the directory layout, from `metastore.partitioned-table`.
+ bool MetastorePartitionedTable() const;
+
+ /// Compression the data files of a format table are written with. It is resolved from
+ /// `file.compression`, then `format-table.file.compression`, then the bare `compression` key
+ /// an engine's own writer reads, then what the table's format writes by default.
+ std::string FormatTableFileCompression() const;
+
bool GlobalIndexEnabled() const;
Result<std::optional<std::string>> CreateGlobalIndexExternalPath() const;
diff --git a/src/paimon/core/core_options_test.cpp b/src/paimon/core/core_options_test.cpp
index 7bd5b5e7..165c2661 100644
--- a/src/paimon/core/core_options_test.cpp
+++ b/src/paimon/core/core_options_test.cpp
@@ -605,6 +605,63 @@ TEST(CoreOptionsTest, TestLookupCompactMaxIntervalComputedValue) {
ASSERT_EQ(13, core_options.GetLookupCompactMaxInterval());
}
+TEST(CoreOptionsTest, TestFormatTableOptions) {
+ // A format table reads these through `CoreOptions` like every other option, so their defaults
+ // are asserted here rather than a second time in the format table tests.
+ {
+ ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({}));
+ ASSERT_FALSE(core_options.FileSuffixIncludeCompression());
+ ASSERT_FALSE(core_options.FormatTablePartitionOnlyValueInPath());
+ ASSERT_FALSE(core_options.MetastorePartitionedTable());
+ }
+ {
+ ASSERT_OK_AND_ASSIGN(
+ CoreOptions core_options,
+ CoreOptions::FromMap({{Options::FILE_SUFFIX_INCLUDE_COMPRESSION, "true"},
+ {Options::FORMAT_TABLE_PARTITION_PATH_ONLY_VALUE, "true"},
+ {Options::METASTORE_PARTITIONED_TABLE, "true"}}));
+ ASSERT_TRUE(core_options.FileSuffixIncludeCompression());
+ ASSERT_TRUE(core_options.FormatTablePartitionOnlyValueInPath());
+ ASSERT_TRUE(core_options.MetastorePartitionedTable());
+ }
+}
+
+TEST(CoreOptionsTest, TestFormatTableFileCompression) {
+ // One chain, in the order the rest of the paimon ecosystem resolves it: `file.compression`,
+ // then `format-table.file.compression`, then the bare `compression` key an engine's own
+ // writer reads, then what the format itself writes.
+ {
+ ASSERT_OK_AND_ASSIGN(CoreOptions core_options,
+ CoreOptions::FromMap({{Options::FILE_COMPRESSION, "lz4"},
+ {Options::FORMAT_TABLE_FILE_COMPRESSION, "none"},
+ {"compression", "zstd"}}));
+ ASSERT_EQ(core_options.FormatTableFileCompression(), "lz4");
+ }
+ {
+ ASSERT_OK_AND_ASSIGN(CoreOptions core_options,
+ CoreOptions::FromMap({{Options::FORMAT_TABLE_FILE_COMPRESSION, "lz4"},
+ {"compression", "zstd"}}));
+ ASSERT_EQ(core_options.FormatTableFileCompression(), "lz4");
+ }
+ {
+ ASSERT_OK_AND_ASSIGN(CoreOptions core_options,
+ CoreOptions::FromMap({{"compression", "lz4"}}));
+ ASSERT_EQ(core_options.FormatTableFileCompression(), "lz4");
+ }
+ // Left to the format: parquet writes snappy, orc zstd. This differs from
+ // `GetFileCompression()`, a managed table's `file.compression`, which defaults to zstd.
+ {
+ ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({}));
+ ASSERT_EQ(core_options.FormatTableFileCompression(), "snappy");
+ ASSERT_EQ(core_options.GetFileCompression(), "zstd");
+ }
+ {
+ ASSERT_OK_AND_ASSIGN(CoreOptions core_options,
+ CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}}));
+ ASSERT_EQ(core_options.FormatTableFileCompression(), "zstd");
+ }
+}
+
TEST(CoreOptionsTest, TestDynamicPartitionOverwriteOption) {
{
ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({}));
diff --git a/src/paimon/core/operation/abstract_split_read.cpp b/src/paimon/core/operation/abstract_split_read.cpp
index 3effa5c1..0a33c7e7 100644
--- a/src/paimon/core/operation/abstract_split_read.cpp
+++ b/src/paimon/core/operation/abstract_split_read.cpp
@@ -32,6 +32,8 @@
#include "paimon/common/data/shredding/map_shared_shredding_utils.h"
#include "paimon/common/data/shredding/shredding_file_reader.h"
#include "paimon/common/data/variant/variant_shredding_read_plan_factory.h"
+#include "paimon/common/data/variant/variant_type_utils.h"
+#include "paimon/common/reader/data_file_reader_factory.h"
#include "paimon/common/reader/delegating_prefetch_reader.h"
#include "paimon/common/reader/late_materializing_reader_builder.h"
#include "paimon/common/reader/predicate_batch_reader.h"
@@ -125,30 +127,26 @@ Result<std::unique_ptr<BatchReader>> AbstractSplitRead::ApplyPredicateFilterIfNe
return PredicateBatchReader::Create(std::move(reader), predicate, arrow_pool_);
}
+DataFileReadOptions AbstractSplitRead::BuildDataFileReadOptions() const {
+ DataFileReadOptions read_options;
+ read_options.cache = options_.GetCache();
+ read_options.read_batch_size = options_.GetReadBatchSize();
+ read_options.prefetch_enabled = context_->EnablePrefetch();
+ read_options.prefetch_max_parallel_num = context_->GetPrefetchMaxParallelNum();
+ read_options.prefetch_batch_count = context_->GetPrefetchBatchCount();
+ read_options.adaptive_prefetch_strategy = options_.EnableAdaptivePrefetchStrategy();
+ read_options.read_ahead_cache_enabled = context_->ReadAheadCacheEnabled();
+ read_options.cache_config = context_->GetCacheConfig();
+ read_options.prefetch_io_metrics_enabled = options_.PrefetchIoMetricsEnabled();
+ return read_options;
+}
+
Result<std::unique_ptr<ReaderBuilder>> AbstractSplitRead::PrepareReaderBuilder(
const std::string& format_identifier,
const std::map<std::string, std::string>& extra_format_options) const {
- std::map<std::string, std::string> format_options = options_.ToMap();
- // The blob placeholder channels are internal: strip user-supplied blob.internal.* table
- // options so only the internal read path can enable them through extra_format_options.
- BlobDefs::EraseInternalPlaceholderOptions(&format_options);
- for (const auto& [key, value] : extra_format_options) {
- format_options[key] = value;
- }
- PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileFormat> file_format,
- FileFormatFactory::Get(format_identifier, format_options));
- PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<ReaderBuilder> reader_builder,
- file_format->CreateReaderBuilder(options_.GetReadBatchSize()));
- reader_builder->WithMemoryPool(pool_);
- reader_builder->WithCache(options_.GetCache());
- // Propagate the framework runtime read state so each format can adapt its own
- // behavior (e.g. parquet disabling its pre-buffer when the shared read-ahead cache
- // takes over prefetching), instead of mutating format options here.
- ReadHints read_hints;
- read_hints.prefetch_enabled = context_->EnablePrefetch();
- read_hints.read_ahead_cache_enabled = context_->ReadAheadCacheEnabled();
- reader_builder->WithReadHints(read_hints);
- return reader_builder;
+ return DataFileReaderFactory::CreateReaderBuilder(format_identifier, options_.ToMap(),
+ extra_format_options,
+ BuildDataFileReadOptions(), pool_);
}
Result<std::unique_ptr<FileBatchReader>> AbstractSplitRead::CreateFileBatchReader(
@@ -158,25 +156,9 @@ Result<std::unique_ptr<FileBatchReader>> AbstractSplitRead::CreateFileBatchReade
reader_builder = std::make_unique<LateMaterializingReaderBuilder>(std::move(reader_builder),
arrow_pool_);
}
- // TODO(xinyu.lxy): test format table for mosaic format
- if (context_->EnablePrefetch() && file_format_identifier != "blob" &&
- file_format_identifier != "avro" && file_format_identifier != "mosaic") {
- PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<PrefetchFileBatchReaderImpl> prefetch_reader,
- PrefetchFileBatchReaderImpl::Create(
- data_file_path, data_file_size, reader_builder.get(),
- options_.GetFileSystem(), context_->GetPrefetchMaxParallelNum(),
- options_.GetReadBatchSize(), context_->GetPrefetchBatchCount(),
- options_.EnableAdaptivePrefetchStrategy(), executor_,
- /*initialize_read_ranges=*/false,
- context_->ReadAheadCacheEnabled(), context_->GetCacheConfig(),
- options_.PrefetchIoMetricsEnabled(), pool_, arrow_pool_));
- return std::make_unique<DelegatingPrefetchReader>(std::move(prefetch_reader));
- } else {
- PAIMON_ASSIGN_OR_RAISE(
- std::shared_ptr<InputStream> input_stream,
- options_.GetFileSystem()->Open(FileStatus(data_file_path, data_file_size)));
- return reader_builder->Build(input_stream);
- }
+ return DataFileReaderFactory::Open(file_format_identifier, data_file_path, data_file_size,
+ reader_builder.get(), BuildDataFileReadOptions(),
+ options_.GetFileSystem(), executor_, pool_, arrow_pool_);
}
Result<std::unique_ptr<FileBatchReader>> AbstractSplitRead::CreateFieldMappingReader(
diff --git a/src/paimon/core/operation/abstract_split_read.h b/src/paimon/core/operation/abstract_split_read.h
index 5a3f3068..1e5f5644 100644
--- a/src/paimon/core/operation/abstract_split_read.h
+++ b/src/paimon/core/operation/abstract_split_read.h
@@ -27,6 +27,7 @@
#include <vector>
#include "arrow/type_fwd.h"
+#include "paimon/common/reader/data_file_reader_factory.h"
#include "paimon/core/core_options.h"
#include "paimon/core/deletionvectors/deletion_vector.h"
#include "paimon/core/io/field_mapping_reader.h"
@@ -102,6 +103,10 @@ class AbstractSplitRead : public SplitRead {
const std::optional<std::vector<std::string>>& write_cols);
private:
+ /// What a data file is opened with, gathered from the context and the table options once so
+ /// that the reader builder and the file reader cannot be built from different answers.
+ DataFileReadOptions BuildDataFileReadOptions() const;
+
Result<std::unique_ptr<ReaderBuilder>> PrepareReaderBuilder(
const std::string& format_identifier,
const std::map<std::string, std::string>& extra_format_options) const;
diff --git a/src/paimon/core/operation/commit_context.cpp b/src/paimon/core/operation/commit_context.cpp
index fe98095d..2b62fdc2 100644
--- a/src/paimon/core/operation/commit_context.cpp
+++ b/src/paimon/core/operation/commit_context.cpp
@@ -26,6 +26,7 @@
#include "paimon/memory/memory_pool.h"
#include "paimon/result.h"
#include "paimon/status.h"
+#include "paimon/table/format/format_table.h"
namespace paimon {
@@ -35,7 +36,8 @@ CommitContext::CommitContext(const std::string& root_path, const std::string& co
const std::shared_ptr<MemoryPool>& memory_pool,
const std::shared_ptr<Executor>& executor,
const std::shared_ptr<FileSystem>& specific_file_system,
- const std::map<std::string, std::string>& options)
+ const std::map<std::string, std::string>& options,
+ const std::shared_ptr<FormatTable>& format_table)
: root_path_(root_path),
commit_user_(commit_user),
ignore_empty_commit_(ignore_empty_commit),
@@ -44,7 +46,8 @@ CommitContext::CommitContext(const std::string& root_path, const std::string& co
memory_pool_(memory_pool),
executor_(executor),
specific_file_system_(specific_file_system),
- options_(options) {}
+ options_(options),
+ format_table_(format_table) {}
CommitContext::~CommitContext() = default;
@@ -64,6 +67,10 @@ class CommitContextBuilder::Impl {
private:
std::string root_path_;
+ /// Kept across `Reset()`, as `root_path_` is: both name the table this builder builds for,
+ /// rather than a setting of one commit to it.
+ std::shared_ptr<FormatTable> format_table_;
+ bool built_from_format_table_ = false;
std::string commit_user_;
bool ignore_empty_commit_ = true;
bool use_rest_catalog_commit_ = false;
@@ -81,6 +88,15 @@ CommitContextBuilder::CommitContextBuilder(const std::string& root_path,
impl_->commit_user_ = commit_user;
}
+CommitContextBuilder::CommitContextBuilder(const std::shared_ptr<FormatTable>& table)
+ : impl_(std::make_unique<Impl>()) {
+ impl_->format_table_ = table;
+ impl_->built_from_format_table_ = true;
+ if (table != nullptr) {
+ impl_->root_path_ = table->Location();
+ }
+}
+
CommitContextBuilder::~CommitContextBuilder() = default;
CommitContextBuilder& CommitContextBuilder::AddOption(const std::string& key,
@@ -130,6 +146,16 @@ CommitContextBuilder& CommitContextBuilder::WithFileSystem(
}
Result<std::unique_ptr<CommitContext>> CommitContextBuilder::Finish() {
+ if (impl_->built_from_format_table_ && impl_->format_table_ == nullptr) {
+ return Status::Invalid("cannot commit with null format table");
+ }
+ // The table already carries the file system it was loaded through, and from a source this
+ // cannot see behind, so a second answer is refused rather than silently dropped.
+ if (impl_->format_table_ != nullptr && impl_->specific_file_system_ != nullptr) {
+ return Status::Invalid(
+ "a format table carries the file system it was loaded through, so WithFileSystem() "
+ "cannot be used with one");
+ }
PAIMON_ASSIGN_OR_RAISE(impl_->root_path_, PathUtil::NormalizePath(impl_->root_path_));
if (impl_->root_path_.empty()) {
return Status::Invalid("root path is empty");
@@ -137,7 +163,7 @@ Result<std::unique_ptr<CommitContext>> CommitContextBuilder::Finish() {
auto ctx = std::make_unique<CommitContext>(
impl_->root_path_, impl_->commit_user_, impl_->ignore_empty_commit_,
impl_->use_rest_catalog_commit_, impl_->append_commit_check_conflict_, impl_->memory_pool_,
- impl_->executor_, impl_->specific_file_system_, impl_->options_);
+ impl_->executor_, impl_->specific_file_system_, impl_->options_, impl_->format_table_);
impl_->Reset();
return ctx;
}
diff --git a/src/paimon/core/operation/file_store_commit.cpp b/src/paimon/core/operation/file_store_commit.cpp
index 6d900560..c2f949ec 100644
--- a/src/paimon/core/operation/file_store_commit.cpp
+++ b/src/paimon/core/operation/file_store_commit.cpp
@@ -20,6 +20,7 @@
#include "paimon/file_store_commit.h"
#include <cassert>
+#include <optional>
#include <utility>
#include "paimon/commit_context.h"
@@ -37,12 +38,16 @@
#include "paimon/core/operation/key_value_file_store_scan.h"
#include "paimon/core/schema/schema_manager.h"
#include "paimon/core/schema/table_schema.h"
+#include "paimon/core/table/format/format_table_file_store_commit.h"
+#include "paimon/core/table/format/format_table_loader.h"
+#include "paimon/core/utils/branch_manager.h"
#include "paimon/core/utils/field_mapping.h"
#include "paimon/core/utils/file_store_path_factory.h"
#include "paimon/core/utils/snapshot_manager.h"
#include "paimon/format/file_format.h"
#include "paimon/fs/file_system.h"
#include "paimon/result.h"
+#include "paimon/table/format/format_table.h"
namespace arrow {
class Schema;
@@ -92,6 +97,30 @@ CommitScanner::ScanSupplier CreatePkScanSupplier(
};
}
+/// Maps a `CommitContext` onto `FormatTableFileStoreCommit`, refusing by name what a format table
+/// cannot honour rather than silently dropping it. Each of the three is refused only when set away
+/// from its default, so an ordinary commit is unaffected.
+Result<std::unique_ptr<FileStoreCommit>> NewFormatTableCommit(
+ const std::shared_ptr<FormatTable>& table, const CommitContext& ctx) {
+ if (!ctx.IgnoreEmptyCommit()) {
+ return Status::NotImplemented(
+ "a format table cannot record an empty commit: keeping one means writing a "
+ "snapshot that adds no files, and there are no snapshots here");
+ }
+ if (ctx.UseRESTCatalogCommit()) {
+ return Status::NotImplemented(
+ "a format table commits by renaming files into place, not by sending a snapshot "
+ "to a rest catalog");
+ }
+ if (ctx.AppendCommitCheckConflict()) {
+ return Status::NotImplemented(
+ "a format table has no manifests to check a concurrent commit against");
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FormatTableFileStoreCommit> format_commit,
+ FormatTableFileStoreCommit::Create(table));
+ return std::unique_ptr<FileStoreCommit>(std::move(format_commit));
+}
+
} // namespace
Result<std::unique_ptr<FileStoreCommit>> FileStoreCommit::Create(
@@ -106,16 +135,36 @@ Result<std::unique_ptr<FileStoreCommit>> FileStoreCommit::Create(
return Status::Invalid("executor is null pointer");
}
+ // A table the caller already loaded says what it is, so nothing is read to find out.
+ if (ctx->GetFormatTable() != nullptr) {
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FormatTable> given_table,
+ FormatTable::Copy(ctx->GetFormatTable(), ctx->GetOptions()));
+ return NewFormatTableCommit(given_table, *ctx);
+ }
+
PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options,
CoreOptions::FromMap(ctx->GetOptions(), ctx->GetSpecificFileSystem()));
const std::string& root_path = ctx->GetRootPath();
+ // A format table commits by renaming files into place, so it never reaches the snapshot path
+ // below. The managed path here reads the main branch, so this reads the same one: the two
+ // must not dispatch on different schemas.
auto schema_manager = std::make_shared<SchemaManager>(tmp_options.GetFileSystem(), root_path);
- PAIMON_ASSIGN_OR_RAISE(std::optional<std::shared_ptr<TableSchema>> table_schema,
- schema_manager->Latest());
- if (table_schema == std::nullopt) {
+ std::shared_ptr<TableSchema> latest_schema;
+ PAIMON_ASSIGN_OR_RAISE(
+ std::shared_ptr<FormatTable> format_table,
+ FormatTableLoader::TryLoad(tmp_options.GetFileSystem(), root_path,
+ BranchManager::DEFAULT_MAIN_BRANCH, ctx->GetOptions(),
+ /*specific_table_schema=*/std::nullopt, schema_manager.get(),
+ &latest_schema));
+ if (format_table != nullptr) {
+ return NewFormatTableCommit(format_table, *ctx);
+ }
+ // The schema the dispatch above already read through `schema_manager`, rather than a second
+ // read of the same file.
+ if (latest_schema == nullptr) {
return Status::Invalid("not found latest schema");
}
- const auto& schema = table_schema.value();
+ const std::shared_ptr<TableSchema>& schema = latest_schema;
auto opts = schema->Options();
for (const auto& [key, value] : ctx->GetOptions()) {
opts[key] = value;
@@ -133,11 +182,11 @@ Result<std::unique_ptr<FileStoreCommit>> FileStoreCommit::Create(
return Status::NotImplemented(
"commit operation does not support object store file system for now");
}
- PAIMON_ASSIGN_OR_RAISE(
- std::unique_ptr<BinaryRowPartitionComputer> partition_computer,
- BinaryRowPartitionComputer::Create(
- table_schema.value()->PartitionKeys(), arrow_schema, options.GetPartitionDefaultName(),
- options.LegacyPartitionNameEnabled(), ctx->GetMemoryPool()));
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BinaryRowPartitionComputer> partition_computer,
+ BinaryRowPartitionComputer::Create(schema->PartitionKeys(), arrow_schema,
+ options.GetPartitionDefaultName(),
+ options.LegacyPartitionNameEnabled(),
+ ctx->GetMemoryPool()));
PAIMON_ASSIGN_OR_RAISE(std::vector<std::string> external_paths, options.CreateExternalPaths());
PAIMON_ASSIGN_OR_RAISE(std::optional<std::string> global_index_external_path,
options.CreateGlobalIndexExternalPath());
@@ -145,10 +194,10 @@ Result<std::unique_ptr<FileStoreCommit>> FileStoreCommit::Create(
PAIMON_ASSIGN_OR_RAISE(
std::shared_ptr<FileStorePathFactory> path_factory,
FileStorePathFactory::Create(
- root_path, arrow_schema, table_schema.value()->PartitionKeys(),
- options.GetPartitionDefaultName(), options.GetFileFormat()->Identifier(),
- options.DataFilePrefix(), options.LegacyPartitionNameEnabled(), external_paths,
- global_index_external_path, options.IndexFileInDataFileDir(), ctx->GetMemoryPool()));
+ root_path, arrow_schema, schema->PartitionKeys(), options.GetPartitionDefaultName(),
+ options.GetFileFormat()->Identifier(), options.DataFilePrefix(),
+ options.LegacyPartitionNameEnabled(), external_paths, global_index_external_path,
+ options.IndexFileInDataFileDir(), ctx->GetMemoryPool()));
auto snapshot_manager = std::make_shared<SnapshotManager>(options.GetFileSystem(), root_path);
PAIMON_ASSIGN_OR_RAISE(
@@ -157,9 +206,8 @@ Result<std::unique_ptr<FileStoreCommit>> FileStoreCommit::Create(
options.GetManifestCompression(), path_factory, options.GetCache(),
ctx->GetMemoryPool()));
- PAIMON_ASSIGN_OR_RAISE(
- std::shared_ptr<arrow::Schema> partition_schema,
- FieldMapping::GetPartitionSchema(arrow_schema, table_schema.value()->PartitionKeys()));
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Schema> partition_schema,
+ FieldMapping::GetPartitionSchema(arrow_schema, schema->PartitionKeys()));
PAIMON_ASSIGN_OR_RAISE(
std::shared_ptr<ManifestFile> manifest_file,
ManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(),
@@ -177,22 +225,22 @@ Result<std::unique_ptr<FileStoreCommit>> FileStoreCommit::Create(
options.GetExpireConfig(), options.RealtimeEnabled(), ctx->GetExecutor());
CommitScanner::ScanSupplier scan_supplier;
- if (table_schema.value()->PrimaryKeys().empty()) {
+ if (schema->PrimaryKeys().empty()) {
scan_supplier = CreateAppendScanSupplier(snapshot_manager, schema_manager, manifest_list,
- manifest_file, table_schema.value(), arrow_schema,
- options, ctx->GetExecutor(), ctx->GetMemoryPool());
+ manifest_file, schema, arrow_schema, options,
+ ctx->GetExecutor(), ctx->GetMemoryPool());
} else {
scan_supplier = CreatePkScanSupplier(snapshot_manager, schema_manager, manifest_list,
- manifest_file, table_schema.value(), arrow_schema,
- options, ctx->GetExecutor(), ctx->GetMemoryPool());
+ manifest_file, schema, arrow_schema, options,
+ ctx->GetExecutor(), ctx->GetMemoryPool());
}
return std::make_unique<FileStoreCommitImpl>(
ctx->GetMemoryPool(), ctx->GetExecutor(), arrow_schema, root_path, ctx->GetCommitUser(),
options, path_factory, std::move(partition_computer), snapshot_manager,
ctx->IgnoreEmptyCommit(), ctx->UseRESTCatalogCommit(), ctx->AppendCommitCheckConflict(),
- table_schema.value(), manifest_file, manifest_list, index_manifest_file, expire_snapshots,
- schema_manager, std::move(scan_supplier));
+ schema, manifest_file, manifest_list, index_manifest_file, expire_snapshots, schema_manager,
+ std::move(scan_supplier));
}
} // namespace paimon
diff --git a/src/paimon/core/operation/file_store_write.cpp b/src/paimon/core/operation/file_store_write.cpp
index 74f56dc9..4bd3079d 100644
--- a/src/paimon/core/operation/file_store_write.cpp
+++ b/src/paimon/core/operation/file_store_write.cpp
@@ -41,12 +41,15 @@
#include "paimon/core/schema/schema_manager.h"
#include "paimon/core/schema/table_schema.h"
#include "paimon/core/table/bucket_mode.h"
+#include "paimon/core/table/format/format_table_file_store_write.h"
+#include "paimon/core/table/format/format_table_loader.h"
#include "paimon/core/utils/field_mapping.h"
#include "paimon/core/utils/file_store_path_factory.h"
#include "paimon/core/utils/primary_key_table_utils.h"
#include "paimon/core/utils/snapshot_manager.h"
#include "paimon/format/file_format.h"
#include "paimon/result.h"
+#include "paimon/table/format/format_table.h"
#include "paimon/write_context.h"
namespace arrow {
@@ -77,6 +80,34 @@ Status RestoreRealtimeCommittedProgress(const std::shared_ptr<RealtimeContext>&
return Status::OK();
}
+/// Maps a `WriteContext` onto `FormatTableFileStoreWrite`, refusing by name what a format table
+/// cannot honour rather than silently dropping it.
+Result<std::unique_ptr<FileStoreWrite>> NewFormatTableWrite(
+ const std::shared_ptr<FormatTable>& table, const WriteContext& ctx) {
+ if (ctx.IsStreamingMode()) {
+ return Status::NotImplemented(
+ "a format table has no snapshots, so there is nothing a streaming write could "
+ "commit against");
+ }
+ if (ctx.GetRealtimeContext() != nullptr) {
+ return Status::NotImplemented("a format table has no real-time store to write into");
+ }
+ if (!ctx.GetWriteSchema().empty()) {
+ return Status::NotImplemented(
+ "a format table write takes the table's own columns; a write schema naming a "
+ "subset of them is not supported yet");
+ }
+ // A write id prefixes a postpone-bucket writer's files so that one compaction reader can
+ // put them back in order. A format table has no buckets, so it would identify nothing.
+ if (ctx.GetWriteId().has_value()) {
+ return Status::NotImplemented(
+ "a format table has no buckets, so a write id would name nothing");
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FormatTableFileStoreWrite> format_write,
+ FormatTableFileStoreWrite::Create(table, ctx.GetMemoryPool()));
+ return std::unique_ptr<FileStoreWrite>(std::move(format_write));
+}
+
} // namespace
Result<std::vector<RealtimeCommitProgress>> FileStoreWrite::PrepareCommitWithProgress(int64_t) {
@@ -98,18 +129,36 @@ Result<std::unique_ptr<FileStoreWrite>> FileStoreWrite::Create(std::unique_ptr<W
return Status::Invalid("executor is null pointer");
}
+ // A table the caller already loaded says what it is, so nothing is read to find out.
+ if (ctx->GetFormatTable() != nullptr) {
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FormatTable> given_table,
+ FormatTable::Copy(ctx->GetFormatTable(), ctx->GetOptions()));
+ return NewFormatTableWrite(given_table, *ctx);
+ }
+
PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options,
CoreOptions::FromMap(ctx->GetOptions(), ctx->GetSpecificFileSystem(),
ctx->GetFileSystemSchemeToIdentifierMap()));
std::string branch = ctx->GetBranch();
+ // A format table writes plain data files into a directory, so it never reaches the manifest
+ // path below.
auto schema_manager =
std::make_shared<SchemaManager>(tmp_options.GetFileSystem(), ctx->GetRootPath(), branch);
- PAIMON_ASSIGN_OR_RAISE(std::optional<std::shared_ptr<TableSchema>> table_schema,
- schema_manager->Latest());
- if (table_schema == std::nullopt) {
+ std::shared_ptr<TableSchema> latest_schema;
+ PAIMON_ASSIGN_OR_RAISE(
+ std::shared_ptr<FormatTable> format_table,
+ FormatTableLoader::TryLoad(tmp_options.GetFileSystem(), ctx->GetRootPath(), branch,
+ ctx->GetOptions(), /*specific_table_schema=*/std::nullopt,
+ schema_manager.get(), &latest_schema));
+ if (format_table != nullptr) {
+ return NewFormatTableWrite(format_table, *ctx);
+ }
+ // The schema the dispatch above already read through `schema_manager`, rather than a second
+ // read of the same file.
+ if (latest_schema == nullptr) {
return Status::Invalid(fmt::format("cannot found latest schema in branch {}", branch));
}
- const auto& schema = table_schema.value();
+ const std::shared_ptr<TableSchema>& schema = latest_schema;
auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields());
PAIMON_RETURN_NOT_OK(BlobUtils::ValidateMapBlobWriteSchema(arrow_schema));
auto opts = schema->Options();
diff --git a/src/paimon/core/operation/read_context.cpp b/src/paimon/core/operation/read_context.cpp
index 774f2a5b..807a507b 100644
--- a/src/paimon/core/operation/read_context.cpp
+++ b/src/paimon/core/operation/read_context.cpp
@@ -27,6 +27,7 @@
#include "paimon/executor.h"
#include "paimon/memory/memory_pool.h"
#include "paimon/status.h"
+#include "paimon/table/format/format_table.h"
namespace paimon {
class Predicate;
@@ -43,7 +44,8 @@ ReadContext::ReadContext(
const std::map<std::string, std::string>& fs_scheme_to_identifier_map,
const std::shared_ptr<RealtimeContext>& realtime_context,
const std::map<std::string, std::string>& options, bool read_ahead_cache_enabled,
- const CacheConfig& cache_config, const std::shared_ptr<Cache>& cache)
+ const CacheConfig& cache_config, const std::shared_ptr<Cache>& cache,
+ const std::shared_ptr<FormatTable>& format_table)
: path_(path),
branch_(branch),
read_field_names_(read_field_names),
@@ -65,7 +67,8 @@ ReadContext::ReadContext(
options_(options),
read_ahead_cache_enabled_(read_ahead_cache_enabled),
cache_config_(cache_config),
- cache_(cache) {}
+ cache_(cache),
+ format_table_(format_table) {}
ReadContext::~ReadContext() {
if (read_schema_ && read_schema_->release) {
@@ -115,6 +118,10 @@ class ReadContextBuilder::Impl {
private:
std::string path_;
+ /// Kept across `Reset()`, as `path_` is: both name the table this builder builds for, rather
+ /// than a setting of one read of it.
+ std::shared_ptr<FormatTable> format_table_;
+ bool built_from_format_table_ = false;
std::string branch_ = BranchManager::DEFAULT_MAIN_BRANCH;
std::vector<std::string> read_field_names_;
std::vector<int32_t> read_field_ids_;
@@ -144,6 +151,15 @@ ReadContextBuilder::ReadContextBuilder(const std::string& path)
impl_->path_ = path;
}
+ReadContextBuilder::ReadContextBuilder(const std::shared_ptr<FormatTable>& table)
+ : impl_(std::make_unique<ReadContextBuilder::Impl>()) {
+ impl_->format_table_ = table;
+ impl_->built_from_format_table_ = true;
+ if (table != nullptr) {
+ impl_->path_ = table->Location();
+ }
+}
+
ReadContextBuilder::~ReadContextBuilder() = default;
ReadContextBuilder::ReadContextBuilder(ReadContextBuilder&&) noexcept = default;
@@ -274,6 +290,28 @@ ReadContextBuilder& ReadContextBuilder::WithCache(const std::shared_ptr<Cache>&
}
Result<std::unique_ptr<ReadContext>> ReadContextBuilder::Finish() {
+ if (impl_->built_from_format_table_ && impl_->format_table_ == nullptr) {
+ return Status::Invalid("cannot read with null format table");
+ }
+ if (impl_->format_table_ != nullptr) {
+ // The table already answers each of these, and from a source this cannot see behind, so a
+ // second answer is refused rather than silently dropped.
+ if (impl_->table_schema_) {
+ return Status::Invalid(
+ "a format table carries its own schema, so SetTableSchema() cannot be used with "
+ "one");
+ }
+ if (impl_->specific_file_system_ != nullptr ||
+ !impl_->fs_scheme_to_identifier_map_.empty()) {
+ return Status::Invalid(
+ "a format table carries the file system it was loaded through, so WithFileSystem() "
+ "and WithFileSystemSchemeToIdentifierMap() cannot be used with one");
+ }
+ if (impl_->branch_ != BranchManager::DEFAULT_MAIN_BRANCH) {
+ return Status::Invalid(
+ "a format table has no branches, so WithBranch() cannot be used with one");
+ }
+ }
PAIMON_ASSIGN_OR_RAISE(impl_->path_, PathUtil::NormalizePath(impl_->path_));
if (impl_->path_.empty()) {
return Status::Invalid("cannot read with empty table path");
@@ -312,7 +350,7 @@ Result<std::unique_ptr<ReadContext>> ReadContextBuilder::Finish() {
impl_->row_to_batch_thread_number_, impl_->table_schema_, impl_->memory_pool_,
impl_->executor_, impl_->specific_file_system_, impl_->fs_scheme_to_identifier_map_,
impl_->realtime_context_, impl_->options_, impl_->read_ahead_cache_enabled_,
- impl_->cache_config_, impl_->cache_);
+ impl_->cache_config_, impl_->cache_, impl_->format_table_);
if (impl_->read_schema_ && impl_->read_schema_->release) {
ctx->SetReadSchema(std::move(impl_->read_schema_));
}
diff --git a/src/paimon/core/operation/scan_context.cpp b/src/paimon/core/operation/scan_context.cpp
index 0b2ab855..6bc80fe7 100644
--- a/src/paimon/core/operation/scan_context.cpp
+++ b/src/paimon/core/operation/scan_context.cpp
@@ -24,6 +24,7 @@
#include "paimon/executor.h"
#include "paimon/memory/memory_pool.h"
#include "paimon/status.h"
+#include "paimon/table/format/format_table.h"
namespace paimon {
class Predicate;
@@ -38,7 +39,8 @@ ScanContext::ScanContext(const std::string& path, bool is_streaming_mode,
const std::shared_ptr<FileSystem>& specific_file_system,
const std::optional<std::string>& table_schema,
const std::map<std::string, std::string>& options,
- const std::shared_ptr<Cache>& cache)
+ const std::shared_ptr<Cache>& cache,
+ const std::shared_ptr<FormatTable>& format_table)
: path_(path),
is_streaming_mode_(is_streaming_mode),
limit_(limit),
@@ -50,7 +52,8 @@ ScanContext::ScanContext(const std::string& path, bool is_streaming_mode,
specific_file_system_(specific_file_system),
table_schema_(table_schema),
options_(options),
- cache_(cache) {}
+ cache_(cache),
+ format_table_(format_table) {}
ScanContext::~ScanContext() = default;
@@ -76,6 +79,10 @@ class ScanContextBuilder::Impl {
private:
std::string path_;
+ /// Kept across `Reset()`, as `path_` is: both name the table this builder builds for, rather
+ /// than a setting of one scan of it.
+ std::shared_ptr<FormatTable> format_table_;
+ bool built_from_format_table_ = false;
bool is_streaming_mode_ = false;
std::optional<int32_t> limit_;
std::optional<int32_t> bucket_filter_;
@@ -96,6 +103,16 @@ ScanContextBuilder::ScanContextBuilder(const std::string& path)
: impl_(std::make_unique<ScanContextBuilder::Impl>()) {
impl_->path_ = path;
}
+
+ScanContextBuilder::ScanContextBuilder(const std::shared_ptr<FormatTable>& table)
+ : impl_(std::make_unique<ScanContextBuilder::Impl>()) {
+ impl_->format_table_ = table;
+ impl_->built_from_format_table_ = true;
+ if (table != nullptr) {
+ impl_->path_ = table->Location();
+ }
+}
+
ScanContextBuilder::~ScanContextBuilder() = default;
ScanContextBuilder& ScanContextBuilder::WithStreamingMode(bool is_streaming_mode) {
impl_->is_streaming_mode_ = is_streaming_mode;
@@ -175,6 +192,23 @@ ScanContextBuilder& ScanContextBuilder::WithCache(const std::shared_ptr<Cache>&
}
Result<std::unique_ptr<ScanContext>> ScanContextBuilder::Finish() {
+ if (impl_->built_from_format_table_ && impl_->format_table_ == nullptr) {
+ return Status::Invalid("cannot scan with null format table");
+ }
+ if (impl_->format_table_ != nullptr) {
+ // The table already answers both, and from a source this cannot see behind, so a second
+ // answer is refused rather than silently dropped.
+ if (impl_->table_schema_) {
+ return Status::Invalid(
+ "a format table carries its own schema, so SetTableSchema() cannot be used with "
+ "one");
+ }
+ if (impl_->specific_file_system_ != nullptr) {
+ return Status::Invalid(
+ "a format table carries the file system it was loaded through, so WithFileSystem() "
+ "cannot be used with one");
+ }
+ }
PAIMON_ASSIGN_OR_RAISE(impl_->path_, PathUtil::NormalizePath(impl_->path_));
if (impl_->path_.empty()) {
return Status::Invalid("cannot scan with empty table path");
@@ -186,7 +220,8 @@ Result<std::unique_ptr<ScanContext>> ScanContextBuilder::Finish() {
std::make_shared<ScanFilter>(impl_->predicates_, impl_->partition_filters_,
impl_->bucket_filter_),
impl_->global_index_result_, impl_->realtime_context_, impl_->memory_pool_, executor,
- impl_->specific_file_system_, impl_->table_schema_, impl_->options_, impl_->cache_);
+ impl_->specific_file_system_, impl_->table_schema_, impl_->options_, impl_->cache_,
+ impl_->format_table_);
impl_->Reset();
return ctx;
}
diff --git a/src/paimon/core/operation/write_context.cpp b/src/paimon/core/operation/write_context.cpp
index c186a6c8..0b9f8942 100644
--- a/src/paimon/core/operation/write_context.cpp
+++ b/src/paimon/core/operation/write_context.cpp
@@ -28,6 +28,7 @@
#include "paimon/memory/memory_pool.h"
#include "paimon/result.h"
#include "paimon/status.h"
+#include "paimon/table/format/format_table.h"
namespace paimon {
@@ -42,7 +43,8 @@ WriteContext::WriteContext(const std::string& root_path, const std::string& comm
const std::shared_ptr<FileSystem>& specific_file_system,
const std::map<std::string, std::string>& fs_scheme_to_identifier_map,
const std::shared_ptr<RealtimeContext>& realtime_context,
- const std::map<std::string, std::string>& options)
+ const std::map<std::string, std::string>& options,
+ const std::shared_ptr<FormatTable>& format_table)
: root_path_(root_path),
commit_user_(commit_user),
branch_(branch),
@@ -58,7 +60,8 @@ WriteContext::WriteContext(const std::string& root_path, const std::string& comm
specific_file_system_(specific_file_system),
fs_scheme_to_identifier_map_(fs_scheme_to_identifier_map),
realtime_context_(realtime_context),
- options_(options) {}
+ options_(options),
+ format_table_(format_table) {}
WriteContext::~WriteContext() = default;
@@ -85,6 +88,10 @@ class WriteContextBuilder::Impl {
private:
std::string root_path_;
+ /// Kept across `Reset()`, as `root_path_` is: both name the table this builder builds for,
+ /// rather than a setting of one write to it.
+ std::shared_ptr<FormatTable> format_table_;
+ bool built_from_format_table_ = false;
std::string commit_user_;
std::string branch_ = BranchManager::DEFAULT_MAIN_BRANCH;
std::optional<int32_t> write_id_;
@@ -109,6 +116,15 @@ WriteContextBuilder::WriteContextBuilder(const std::string& root_path,
impl_->commit_user_ = commit_user;
}
+WriteContextBuilder::WriteContextBuilder(const std::shared_ptr<FormatTable>& table)
+ : impl_(std::make_unique<Impl>()) {
+ impl_->format_table_ = table;
+ impl_->built_from_format_table_ = true;
+ if (table != nullptr) {
+ impl_->root_path_ = table->Location();
+ }
+}
+
WriteContextBuilder::~WriteContextBuilder() = default;
WriteContextBuilder& WriteContextBuilder::AddOption(const std::string& key,
@@ -194,6 +210,23 @@ WriteContextBuilder& WriteContextBuilder::WithRealtimeContext(
}
Result<std::unique_ptr<WriteContext>> WriteContextBuilder::Finish() {
+ if (impl_->built_from_format_table_ && impl_->format_table_ == nullptr) {
+ return Status::Invalid("cannot write with null format table");
+ }
+ if (impl_->format_table_ != nullptr) {
+ // The table already answers each of these, and from a source this cannot see behind, so a
+ // second answer is refused rather than silently dropped.
+ if (impl_->specific_file_system_ != nullptr ||
+ !impl_->fs_scheme_to_identifier_map_.empty()) {
+ return Status::Invalid(
+ "a format table carries the file system it was loaded through, so WithFileSystem() "
+ "and WithFileSystemSchemeToIdentifierMap() cannot be used with one");
+ }
+ if (impl_->branch_ != BranchManager::DEFAULT_MAIN_BRANCH) {
+ return Status::Invalid(
+ "a format table has no branches, so WithBranch() cannot be used with one");
+ }
+ }
PAIMON_ASSIGN_OR_RAISE(impl_->root_path_, PathUtil::NormalizePath(impl_->root_path_));
if (impl_->root_path_.empty()) {
return Status::Invalid("root path is empty");
@@ -210,7 +243,8 @@ Result<std::unique_ptr<WriteContext>> WriteContextBuilder::Finish() {
impl_->ignore_num_bucket_check_, impl_->ignore_previous_files_, enable_multi_thread_spill,
impl_->write_id_, impl_->branch_, impl_->write_schema_, impl_->memory_pool_,
impl_->executor_, impl_->temp_directory_, impl_->specific_file_system_,
- impl_->fs_scheme_to_identifier_map_, impl_->realtime_context_, impl_->options_);
+ impl_->fs_scheme_to_identifier_map_, impl_->realtime_context_, impl_->options_,
+ impl_->format_table_);
impl_->Reset();
return ctx;
}
diff --git a/src/paimon/core/options/table_type.h b/src/paimon/core/options/table_type.h
new file mode 100644
index 00000000..c4233574
--- /dev/null
+++ b/src/paimon/core/options/table_type.h
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <map>
+#include <string>
+
+#include "fmt/format.h"
+#include "paimon/common/utils/string_utils.h"
+#include "paimon/defs.h"
+#include "paimon/result.h"
+#include "paimon/status.h"
+
+namespace paimon {
+
+/// Type of a table, carried by the `type` table option.
+///
+/// Every type paimon defines is named here, even the ones this library cannot open: telling such
+/// a table from a managed one means knowing its name.
+enum class TableType {
+ /// A managed paimon table with snapshots and manifests.
+ TABLE = 1,
+ /// A directory of data files laid out like a standard Hive table, without paimon metadata.
+ FORMAT_TABLE = 2,
+ /// A managed paimon table that also carries the SQL it materializes.
+ MATERIALIZED_TABLE = 3,
+ /// A managed paimon table over the objects of a location.
+ OBJECT_TABLE = 4,
+ /// A lance table, see 'https://lancedb.github.io/lance/'.
+ LANCE_TABLE = 5,
+ /// An iceberg table, see 'https://iceberg.apache.org/'.
+ ICEBERG_TABLE = 6,
+};
+
+/// Identifiers of `TableType` as they appear in the `type` table option.
+struct TableTypeDefine {
+ static constexpr char kTable[] = "table";
+ static constexpr char kFormatTable[] = "format-table";
+ static constexpr char kMaterializedTable[] = "materialized-table";
+ static constexpr char kObjectTable[] = "object-table";
+ static constexpr char kLanceTable[] = "lance-table";
+ static constexpr char kIcebergTable[] = "iceberg-table";
+
+ /// Reads the table type out of a table option map.
+ ///
+ /// An absent `type` is `TableType::TABLE`, and the value is matched without regard to case.
+ /// A value that names no table type is rejected rather than read as a managed table.
+ static Result<TableType> FromOptions(const std::map<std::string, std::string>& options) {
+ auto iter = options.find(Options::TYPE);
+ if (iter == options.end()) {
+ return TableType::TABLE;
+ }
+ std::string value = StringUtils::ToLowerCase(iter->second);
+ if (value == kTable) {
+ return TableType::TABLE;
+ }
+ if (value == kFormatTable) {
+ return TableType::FORMAT_TABLE;
+ }
+ if (value == kMaterializedTable) {
+ return TableType::MATERIALIZED_TABLE;
+ }
+ if (value == kObjectTable) {
+ return TableType::OBJECT_TABLE;
+ }
+ if (value == kLanceTable) {
+ return TableType::LANCE_TABLE;
+ }
+ if (value == kIcebergTable) {
+ return TableType::ICEBERG_TABLE;
+ }
+ return Status::Invalid(fmt::format("unknown table type: {}", iter->second));
+ }
+
+ static Result<bool> IsFormatTable(const std::map<std::string, std::string>& options) {
+ PAIMON_ASSIGN_OR_RAISE(TableType table_type, FromOptions(options));
+ return table_type == TableType::FORMAT_TABLE;
+ }
+};
+
+} // namespace paimon
diff --git a/src/paimon/core/options/table_type_test.cpp b/src/paimon/core/options/table_type_test.cpp
new file mode 100644
index 00000000..6edfacce
--- /dev/null
+++ b/src/paimon/core/options/table_type_test.cpp
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/core/options/table_type.h"
+
+#include <map>
+#include <string>
+
+#include "gtest/gtest.h"
+#include "paimon/defs.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+
+TEST(TableTypeTest, TestEveryNameReadsBackAsItsType) {
+ // Every type paimon defines is named here, including the ones this library cannot open:
+ // telling such a table from a managed one means knowing its name. `materialized-table` is the
+ // one that matters most to get right, since it is the only type besides `table` that the
+ // managed path accepts - a wrong name there would refuse every materialized table.
+ const std::map<std::string, TableType> by_name = {
+ {TableTypeDefine::kTable, TableType::TABLE},
+ {TableTypeDefine::kFormatTable, TableType::FORMAT_TABLE},
+ {TableTypeDefine::kMaterializedTable, TableType::MATERIALIZED_TABLE},
+ {TableTypeDefine::kObjectTable, TableType::OBJECT_TABLE},
+ {TableTypeDefine::kLanceTable, TableType::LANCE_TABLE},
+ {TableTypeDefine::kIcebergTable, TableType::ICEBERG_TABLE}};
+ for (const auto& [name, expected] : by_name) {
+ SCOPED_TRACE(name);
+ ASSERT_OK_AND_ASSIGN(TableType table_type,
+ TableTypeDefine::FromOptions({{Options::TYPE, name}}));
+ ASSERT_EQ(table_type, expected);
+ }
+}
+
+TEST(TableTypeTest, TestAnAbsentTypeIsAManagedTable) {
+ // Almost every table ever written omits the option, so reading it as anything else would
+ // refuse them all.
+ ASSERT_OK_AND_ASSIGN(TableType table_type, TableTypeDefine::FromOptions({}));
+ ASSERT_EQ(table_type, TableType::TABLE);
+ ASSERT_OK_AND_ASSIGN(TableType with_others,
+ TableTypeDefine::FromOptions({{Options::FILE_FORMAT, "parquet"}}));
+ ASSERT_EQ(with_others, TableType::TABLE);
+}
+
+TEST(TableTypeTest, TestATypeIsMatchedIgnoringCase) {
+ for (const char* spelling : {"Format-Table", "FORMAT-TABLE", "format-TABLE"}) {
+ SCOPED_TRACE(spelling);
+ ASSERT_OK_AND_ASSIGN(TableType table_type,
+ TableTypeDefine::FromOptions({{Options::TYPE, spelling}}));
+ ASSERT_EQ(table_type, TableType::FORMAT_TABLE);
+ }
+}
+
+TEST(TableTypeTest, TestAValueNamingNoTypeIsRejected) {
+ // Refused rather than read as a managed table: opening one as managed would look for
+ // snapshots it never had. The value is quoted back as it was given.
+ ASSERT_NOK_WITH_MSG(TableTypeDefine::FromOptions({{Options::TYPE, "nonesuch"}}),
+ "unknown table type: nonesuch");
+ ASSERT_NOK_WITH_MSG(TableTypeDefine::FromOptions({{Options::TYPE, ""}}), "unknown table type");
+}
+
+TEST(TableTypeTest, TestIsFormatTableAnswersForTheFormatTypeAlone) {
+ ASSERT_OK_AND_ASSIGN(bool is_format, TableTypeDefine::IsFormatTable(
+ {{Options::TYPE, TableTypeDefine::kFormatTable}}));
+ ASSERT_TRUE(is_format);
+ for (const char* other : {TableTypeDefine::kTable, TableTypeDefine::kMaterializedTable,
+ TableTypeDefine::kObjectTable}) {
+ SCOPED_TRACE(other);
+ ASSERT_OK_AND_ASSIGN(bool answered,
+ TableTypeDefine::IsFormatTable({{Options::TYPE, other}}));
+ ASSERT_FALSE(answered);
+ }
+ // The absent case goes through `FromOptions()` too, so it is a managed table here as well.
+ ASSERT_OK_AND_ASSIGN(bool absent, TableTypeDefine::IsFormatTable({}));
+ ASSERT_FALSE(absent);
+}
+
+} // namespace paimon::test
diff --git a/src/paimon/core/schema/schema_manager.cpp b/src/paimon/core/schema/schema_manager.cpp
index 2425cc4a..ede262f2 100644
--- a/src/paimon/core/schema/schema_manager.cpp
+++ b/src/paimon/core/schema/schema_manager.cpp
@@ -22,10 +22,12 @@
#include <algorithm>
#include <utility>
+#include "fmt/format.h"
#include "paimon/common/utils/path_util.h"
#include "paimon/core/schema/schema_validation.h"
#include "paimon/core/utils/branch_manager.h"
#include "paimon/core/utils/file_utils.h"
+#include "paimon/defs.h"
#include "paimon/fs/file_system.h"
#include "paimon/status.h"
@@ -110,7 +112,8 @@ Result<std::unique_ptr<TableSchema>> SchemaManager::CreateTable(
PAIMON_ASSIGN_OR_RAISE(
std::unique_ptr<TableSchema> table_schema,
TableSchema::Create(/*schema_id=*/0, schema, partition_keys, primary_keys, options));
- PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateTableSchema(*table_schema));
+ // Shared with every other catalog, so a schema this one persists is one they all open.
+ PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateNewTableSchema(*table_schema, file_system_));
std::string schema_path = ToSchemaPath(0);
PAIMON_ASSIGN_OR_RAISE(std::string content, table_schema->ToJsonString());
auto status = file_system_->AtomicStore(schema_path, content);
diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp
index 26d4100c..840653fe 100644
--- a/src/paimon/core/schema/schema_validation.cpp
+++ b/src/paimon/core/schema/schema_validation.cpp
@@ -23,6 +23,7 @@
#include <cassert>
#include <functional>
#include <map>
+#include <memory>
#include <optional>
#include <set>
#include <sstream>
@@ -30,6 +31,8 @@
#include <unordered_set>
#include <utility>
+#include "arrow/c/bridge.h"
+#include "arrow/c/helpers.h"
#include "arrow/type.h"
#include "fmt/format.h"
#include "fmt/ranges.h"
@@ -38,22 +41,29 @@
#include "paimon/common/data/variant/variant_type_utils.h"
#include "paimon/common/table/special_fields.h"
#include "paimon/common/types/data_field.h"
+#include "paimon/common/utils/arrow/status_utils.h"
#include "paimon/common/utils/arrow/vector_utils.h"
+#include "paimon/common/utils/binary_row_partition_computer.h"
#include "paimon/common/utils/checked_cast.h"
#include "paimon/common/utils/object_utils.h"
+#include "paimon/common/utils/options_utils.h"
#include "paimon/common/utils/preconditions.h"
+#include "paimon/common/utils/scope_guard.h"
#include "paimon/common/utils/string_utils.h"
#include "paimon/core/core_options.h"
#include "paimon/core/options/changelog_producer.h"
#include "paimon/core/options/expire_config.h"
#include "paimon/core/options/map_storage_layout.h"
#include "paimon/core/options/merge_engine.h"
+#include "paimon/core/options/table_type.h"
#include "paimon/core/schema/arrow_schema_validator.h"
#include "paimon/core/schema/table_schema.h"
#include "paimon/core/table/bucket_mode.h"
+#include "paimon/core/utils/branch_manager.h"
#include "paimon/defs.h"
#include "paimon/format/file_format.h"
#include "paimon/result.h"
+#include "paimon/table/format/format_table.h"
namespace paimon {
namespace {
@@ -145,27 +155,159 @@ bool SchemaValidation::IsComplexType(const std::shared_ptr<arrow::Field>& field)
BlobUtils::IsBlobField(field));
}
-Status SchemaValidation::ValidateTableSchema(const TableSchema& schema) {
- const auto& field_names = schema.FieldNames();
- PAIMON_RETURN_NOT_OK(ValidateNoDuplicateField(schema.BucketKeys(), "bucket key"));
- PAIMON_RETURN_NOT_OK(ValidateNoDuplicateField(schema.PrimaryKeys(), "primary key"));
- PAIMON_RETURN_NOT_OK(ValidateNoDuplicateField(schema.PartitionKeys(), "partition key"));
- PAIMON_RETURN_NOT_OK(
- Preconditions::CheckState(ObjectUtils::ContainsAll(field_names, schema.PartitionKeys()),
- "Table column {} should include all partition fields {}",
- field_names, schema.PartitionKeys()));
- PAIMON_RETURN_NOT_OK(
- Preconditions::CheckState(ObjectUtils::ContainsAll(field_names, schema.PrimaryKeys()),
- "Table column {} should include all primary key constraint {}",
- field_names, schema.PrimaryKeys()));
-
- PAIMON_RETURN_NOT_OK(
- ValidateOnlyContainPrimitiveType(schema.Fields(), schema.PrimaryKeys(), "primary key"));
- PAIMON_RETURN_NOT_OK(
- ValidateOnlyContainPrimitiveType(schema.Fields(), schema.PartitionKeys(), "partition"));
+Status SchemaValidation::ValidateGenericSchema(const std::vector<DataField>& fields,
+ const std::vector<std::string>& bucket_keys,
+ const std::vector<std::string>& primary_keys,
+ const std::vector<std::string>& partition_keys) {
+ std::vector<std::string> field_names;
+ field_names.reserve(fields.size());
+ for (const DataField& field : fields) {
+ field_names.push_back(field.Name());
+ }
+ PAIMON_RETURN_NOT_OK(ValidateNoDuplicateField(bucket_keys, "bucket key"));
+ PAIMON_RETURN_NOT_OK(ValidateNoDuplicateField(primary_keys, "primary key"));
+ PAIMON_RETURN_NOT_OK(ValidateNoDuplicateField(partition_keys, "partition key"));
+ PAIMON_RETURN_NOT_OK(Preconditions::CheckState(
+ ObjectUtils::ContainsAll(field_names, partition_keys),
+ "Table column {} should include all partition fields {}", field_names, partition_keys));
+ PAIMON_RETURN_NOT_OK(Preconditions::CheckState(
+ ObjectUtils::ContainsAll(field_names, primary_keys),
+ "Table column {} should include all primary key constraint {}", field_names, primary_keys));
+ for (const auto& field_name : field_names) {
+ if (SpecialFields::IsSystemField(field_name)) {
+ return Status::Invalid(
+ fmt::format("field name '{}' in schema cannot be special field.", field_name));
+ }
+ }
+ PAIMON_RETURN_NOT_OK(ValidateOnlyContainPrimitiveType(fields, primary_keys, "primary key"));
+ PAIMON_RETURN_NOT_OK(ValidateOnlyContainPrimitiveType(fields, partition_keys, "partition"));
// TODO(lisizhuo.lsz): C++ Paimon do not support timestamp & decimal & float & double type in
// partition keys for now.
- PAIMON_RETURN_NOT_OK(ValidateNotContainSpecificType(schema.Fields(), schema.PartitionKeys()));
+ PAIMON_RETURN_NOT_OK(ValidateNotContainSpecificType(fields, partition_keys));
+ return Status::OK();
+}
+
+Status SchemaValidation::ValidateNewTableSchema(const TableSchema& schema,
+ const std::shared_ptr<FileSystem>& file_system) {
+ const std::map<std::string, std::string>& options = schema.Options();
+ PAIMON_ASSIGN_OR_RAISE(TableType table_type, TableTypeDefine::FromOptions(options));
+ if (table_type != TableType::TABLE && table_type != TableType::MATERIALIZED_TABLE &&
+ table_type != TableType::FORMAT_TABLE) {
+ // Quoted back rather than re-rendered from `table_type`, so the message says what was
+ // actually asked for.
+ auto type_iter = options.find(Options::TYPE);
+ return Status::NotImplemented(fmt::format(
+ "Cannot create a table whose '{}' is '{}': paimon-cpp does not implement this table "
+ "type.",
+ Options::TYPE, type_iter == options.end() ? std::string() : type_iter->second));
+ }
+ if (table_type == TableType::FORMAT_TABLE) {
+ PAIMON_RETURN_NOT_OK(ValidateGenericTableSchema(schema));
+ // At creation the schema's own options are the only ones there are. The caller's file
+ // system is passed through rather than resolved again from them, so validation runs
+ // against the one the table is about to be created on.
+ return ValidateFormatTableSchema(schema, schema.Options(), file_system);
+ }
+ return ValidateTableSchema(schema);
+}
+
+Status SchemaValidation::ValidateGenericTableSchema(const TableSchema& schema) {
+ return ValidateGenericSchema(schema.Fields(), schema.BucketKeys(), schema.PrimaryKeys(),
+ schema.PartitionKeys());
+}
+
+Status SchemaValidation::ValidateGenericDataSchema(const DataSchema& schema) {
+ // A `DataSchema` names its field types through its arrow schema rather than through
+ // `DataField`s, so they are converted back and the same rules run on them.
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> c_schema, schema.GetArrowSchema());
+ ScopeGuard schema_guard([&c_schema]() { ArrowSchemaRelease(c_schema.get()); });
+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> arrow_schema,
+ arrow::ImportSchema(c_schema.get()));
+ PAIMON_ASSIGN_OR_RAISE(std::vector<DataField> fields,
+ DataField::ConvertArrowSchemaToDataFields(arrow_schema));
+ return ValidateGenericSchema(fields, schema.BucketKeys(), schema.PrimaryKeys(),
+ schema.PartitionKeys());
+}
+
+Status SchemaValidation::ValidateFormatTableSchema(
+ const DataSchema& schema, const std::map<std::string, std::string>& effective_options,
+ const std::shared_ptr<FileSystem>& file_system) {
+ // Runs both when the table is created and when it is opened: creation alone would let a
+ // schema written elsewhere through, and opening alone would persist a table nothing can load.
+ if (!schema.PrimaryKeys().empty()) {
+ return Status::Invalid(
+ "Cannot define primary keys for a format table: a directory of data files records no "
+ "row identity to merge on.");
+ }
+
+ PAIMON_ASSIGN_OR_RAISE(std::string file_format,
+ OptionsUtils::GetValueFromMap<std::string>(
+ effective_options, Options::FILE_FORMAT, "parquet"));
+ // Before `CoreOptions`, which resolves `file.format` through the format factories and would
+ // fail with a missing-factory error where this names the format. The parsed value is not
+ // kept; this only has to fail for a format nothing here can read.
+ PAIMON_RETURN_NOT_OK(FormatTable::ParseFormat(file_format));
+
+ // The remaining options are read through `CoreOptions`, so a default only ever changes in one
+ // place. `target-file-row-num` is validated there, so it needs no check of its own here.
+ PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options,
+ CoreOptions::FromMap(effective_options, file_system));
+
+ // A format table has no branches: its data is the files under its one location, which is
+ // where a read or a write would go whatever this named. Refused wherever the option comes
+ // from - the schema, a catalog that parsed `tbl$branch_dev`, or one call's options - rather
+ // than quietly ignored, which is the answer `WithBranch()` already gives.
+ // Normalized first, since an empty value names the main branch rather than a branch of its own.
+ const std::string branch = BranchManager::NormalizeBranch(core_options.GetBranch());
+ if (branch != BranchManager::DEFAULT_MAIN_BRANCH) {
+ return Status::Invalid(fmt::format("a format table has no branches, so '{}' cannot be '{}'",
+ Options::BRANCH, branch));
+ }
+
+ if (core_options.FormatTablePartitionOnlyValueInPath() && schema.PartitionKeys().empty()) {
+ return Status::Invalid(
+ "Cannot set 'format-table.partition-path-only-value' on a table with no partition "
+ "keys: the layout names a directory by its partition value alone.");
+ }
+
+ // A table of nothing but partition columns leaves a write nothing to write and a read no
+ // column to count rows by.
+ if (!schema.PartitionKeys().empty() &&
+ schema.PartitionKeys().size() == schema.FieldNames().size()) {
+ return Status::Invalid(
+ "A format table cannot be partitioned by every one of its columns: the data files "
+ "would hold nothing, since partition values live in the directory names.");
+ }
+
+ // A partition value makes the round trip through its column type on the way to a directory
+ // name and back, so a type that cannot leaves a table nothing can read or write. Checked by
+ // building the computer that does the round trip, not by listing the types it accepts, which
+ // would be a second list to keep in step.
+ if (!schema.PartitionKeys().empty()) {
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> c_schema, schema.GetArrowSchema());
+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> arrow_schema,
+ arrow::ImportSchema(c_schema.get()));
+ Result<std::unique_ptr<BinaryRowPartitionComputer>> partition_computer =
+ BinaryRowPartitionComputer::Create(
+ schema.PartitionKeys(), arrow_schema, core_options.GetPartitionDefaultName(),
+ core_options.LegacyPartitionNameEnabled(), GetDefaultPool());
+ if (!partition_computer.ok()) {
+ return Status(partition_computer.status().code(),
+ fmt::format("a format table cannot be partitioned by these columns: {}",
+ partition_computer.status().message()));
+ }
+ }
+
+ if (core_options.MetastorePartitionedTable()) {
+ return Status::NotImplemented(
+ "'metastore.partitioned-table' is not supported by paimon-cpp yet: its partitions "
+ "would come from the catalog rather than from the directories a scan here reads.");
+ }
+ return Status::OK();
+}
+
+Status SchemaValidation::ValidateTableSchema(const TableSchema& schema) {
+ PAIMON_RETURN_NOT_OK(ValidateGenericTableSchema(schema));
PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(schema.Options()));
PAIMON_RETURN_NOT_OK(ValidateBucket(schema, options));
@@ -187,12 +329,6 @@ Status SchemaValidation::ValidateTableSchema(const TableSchema& schema) {
// TODO(yonghao.fyh): check changelog num retain
// TODO(yonghao.fyh): support file format validate data fields
- for (const auto& field_name : field_names) {
- if (SpecialFields::IsSystemField(field_name)) {
- return Status::Invalid(
- fmt::format("field name '{}' in schema cannot be special field.", field_name));
- }
- }
// TODO(yonghao.fyh): check streaming read overwrite
// TODO(yonghao.fyh): check 'partition.expiration-time'
// TODO(yonghao.fyh): check 'rowkind.field'
diff --git a/src/paimon/core/schema/schema_validation.h b/src/paimon/core/schema/schema_validation.h
index 388ab42e..4f6f3cad 100644
--- a/src/paimon/core/schema/schema_validation.h
+++ b/src/paimon/core/schema/schema_validation.h
@@ -20,6 +20,7 @@
#pragma once
#include <cstdint>
+#include <map>
#include <memory>
#include <string>
#include <vector>
@@ -36,6 +37,8 @@ class Field;
namespace paimon {
class CoreOptions;
class DataField;
+class DataSchema;
+class FileSystem;
class TableSchema;
/// Validation utils for `TableSchema`.
@@ -46,11 +49,56 @@ class SchemaValidation {
static Status ValidateTableSchema(const TableSchema& schema);
+ /// Validates a schema a table is about to be created from, whichever catalog is creating it.
+ ///
+ /// It picks the rules by the schema's `type` option: a format table owns no bucket, manifest
+ /// or snapshot machinery, so only the structural invariants apply to it. A type this library
+ /// cannot load at all is refused rather than persisted, since the table would look created
+ /// and fail only when someone tried to open it.
+ ///
+ /// @param file_system The file system the table is being created on, already resolved, or
+ /// null to resolve it from the schema's options. Nothing here reads through it today;
+ /// passing it keeps validation from resolving a different one than creation uses.
+ static Status ValidateNewTableSchema(const TableSchema& schema,
+ const std::shared_ptr<FileSystem>& file_system);
+
+ /// Validates the structural invariants every table shares, whatever its type: partition and
+ /// primary key fields exist and are free of duplicates, key fields are primitive, and no field
+ /// takes a reserved name.
+ static Status ValidateGenericTableSchema(const TableSchema& schema);
+
+ /// The same rules as `ValidateGenericTableSchema()`, for a schema this library did not
+ /// create. A `DataSchema` names its field types through its arrow schema rather than through
+ /// `DataField`s, which is the only reason this is a second entry point.
+ static Status ValidateGenericDataSchema(const DataSchema& schema);
+
+ /// Validates what a format table additionally requires, on top of
+ /// `ValidateGenericTableSchema()`. It takes a `DataSchema` so that it can run both at creation
+ /// and at `FormatTable::Create()`, where the schema may never have passed through creation
+ /// here at all.
+ ///
+ /// @param effective_options The options the table will actually run with: the schema's own at
+ /// creation, and those with anything given at the call merged on top at
+ /// `FormatTable::Create()`. Passing the schema's alone would let an option given at the
+ /// call reach a table that refuses it in its schema.
+ /// @param file_system The file system the caller already resolved, or null when it named one
+ /// through the options. Spares resolving `file-system` a second time, which would fail
+ /// for a caller that handed its own over rather than naming one.
+ static Status ValidateFormatTableSchema(
+ const DataSchema& schema, const std::map<std::string, std::string>& effective_options,
+ const std::shared_ptr<FileSystem>& file_system);
+
static bool IsPostponeBucketTable(const TableSchema& schema, int32_t bucket);
private:
static Status ValidateNoDuplicateField(const std::vector<std::string>& field_names,
const std::string& error_message_intro);
+ /// The rules `ValidateGenericTableSchema()` and `ValidateGenericDataSchema()` share, on the
+ /// fields both can hand over.
+ static Status ValidateGenericSchema(const std::vector<DataField>& fields,
+ const std::vector<std::string>& bucket_keys,
+ const std::vector<std::string>& primary_keys,
+ const std::vector<std::string>& partition_keys);
static Status ValidateOnlyContainPrimitiveType(const std::vector<DataField>& fields,
const std::vector<std::string>& field_names,
const std::string& error_message_intro);
diff --git a/src/paimon/core/table/format/format_commit_message.h b/src/paimon/core/table/format/format_commit_message.h
new file mode 100644
index 00000000..6b607500
--- /dev/null
+++ b/src/paimon/core/table/format/format_commit_message.h
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <map>
+#include <string>
+
+#include "fmt/format.h"
+#include "fmt/ranges.h"
+#include "paimon/commit_message.h"
+
+namespace paimon {
+
+/// One file a `FormatTableWrite` has written but not yet published.
+///
+/// The file is complete on disk under `temp_file_path`, which a scan skips; committing renames it
+/// to `file_path`, which is what makes it part of the table.
+///
+/// It is a `CommitMessage` so that a format table can be written and committed through
+/// `FileStoreWrite` and `FileStoreCommit` like any other. It names a staged path rather than
+/// files to record in a manifest, so `CommitMessage::Serialize()` refuses it: there is no
+/// cross-runtime encoding for one, and a write and its commit belong to the same process.
+struct FormatCommitMessage : public CommitMessage {
+ FormatCommitMessage(const std::string& _temp_file_path, const std::string& _file_path,
+ const std::map<std::string, std::string>& _partition, int64_t _record_count,
+ int64_t _file_size)
+ : temp_file_path(_temp_file_path),
+ file_path(_file_path),
+ partition(_partition),
+ record_count(_record_count),
+ file_size(_file_size) {}
+
+ ~FormatCommitMessage() override = default;
+
+ std::string ToString() const {
+ return fmt::format(
+ "FormatCommitMessage{{file_path={}, temp_file_path={}, partition={}, record_count={}, "
+ "file_size={}}}",
+ file_path, temp_file_path, partition, record_count, file_size);
+ }
+
+ /// Path the data was written to: a hidden file a scan skips.
+ std::string temp_file_path;
+ /// Path the file is renamed to when the write is committed.
+ std::string file_path;
+ /// Partition the file belongs to, empty when the table is not partitioned.
+ std::map<std::string, std::string> partition;
+ int64_t record_count;
+ int64_t file_size;
+};
+
+} // namespace paimon
diff --git a/src/paimon/core/table/format/format_data_split.h b/src/paimon/core/table/format/format_data_split.h
new file mode 100644
index 00000000..f80e2220
--- /dev/null
+++ b/src/paimon/core/table/format/format_data_split.h
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <limits>
+#include <map>
+#include <string>
+#include <vector>
+
+#include "paimon/table/source/split.h"
+
+namespace paimon {
+
+/// A split of a format table: the data files of one partition directory, or of the table
+/// directory itself when the table is not partitioned.
+///
+/// A file is never divided between splits: parquet and orc record their own row group and stripe
+/// boundaries, which a reader handed a byte range would have to rediscover. In-memory only, so
+/// plan and read within one process.
+struct FormatDataSplit : public Split {
+ struct FileMeta {
+ FileMeta(const std::string& _file_path, int64_t _file_size)
+ : file_path(_file_path), file_size(_file_size) {}
+
+ bool operator==(const FileMeta& other) const {
+ return file_path == other.file_path && file_size == other.file_size;
+ }
+
+ std::string file_path;
+ /// In bytes, as reported by the listing that found it.
+ int64_t file_size;
+ };
+
+ FormatDataSplit(const std::vector<FileMeta>& files,
+ const std::map<std::string, std::string>& partition)
+ : files(files), partition(partition) {}
+
+ ~FormatDataSplit() override = default;
+
+ /// Sum of every file's size. Saturates rather than wrapping, since the sizes are whatever
+ /// the split was given.
+ int64_t TotalSize() const {
+ int64_t total = 0;
+ for (const FileMeta& file : files) {
+ if (file.file_size > 0 &&
+ total > std::numeric_limits<int64_t>::max() - file.file_size) {
+ return std::numeric_limits<int64_t>::max();
+ }
+ total += file.file_size;
+ }
+ return total;
+ }
+
+ /// Data files of this split, read in this order.
+ std::vector<FileMeta> files;
+
+ /// Keyed by partition field name, empty when the table is not partitioned.
+ /// `partition.default-name` stands for a null value.
+ std::map<std::string, std::string> partition;
+};
+
+} // namespace paimon
diff --git a/src/paimon/core/table/format/format_file_listing.cpp b/src/paimon/core/table/format/format_file_listing.cpp
new file mode 100644
index 00000000..c4cf07bb
--- /dev/null
+++ b/src/paimon/core/table/format/format_file_listing.cpp
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/core/table/format/format_file_listing.h"
+
+#include <optional>
+#include <utility>
+
+#include "fmt/format.h"
+#include "paimon/common/utils/path_util.h"
+#include "paimon/core/utils/partition_path_utils.h"
+#include "paimon/fs/file_system.h"
+
+namespace paimon {
+
+bool FormatFileListing::IsReservedDirectory(const std::string& name) {
+ return name == "schema" || name == "branch";
+}
+
+Status FormatFileListing::ListDataFiles(const std::shared_ptr<FileSystem>& file_system,
+ const std::string& root,
+ const FormatDataFileListingOptions& options,
+ std::vector<FormatDataSplit::FileMeta>* files) {
+ // Checked explicitly: the file systems here report a missing directory as an empty listing
+ // rather than as an error.
+ PAIMON_ASSIGN_OR_RAISE(FileStatus root_status, file_system->GetFileStatus(root));
+ if (!root_status.IsDir()) {
+ return Status::Invalid(
+ fmt::format("{} is not a directory, so it holds no table data", root));
+ }
+
+ std::vector<std::string> level = {root};
+ // Depth of the directories in `level`, counted from the listed root.
+ int32_t depth = 0;
+ while (!level.empty()) {
+ // The default partition name holds table content only at a partition level; anywhere else
+ // a hidden name is a staging tree.
+ const bool children_are_partitions = options.partition_levels >= depth + 1;
+ const bool exempt_default_part_name = children_are_partitions &&
+ options.only_value_in_path &&
+ !options.default_part_name.empty();
+ std::vector<std::string> next;
+ for (const std::string& directory : level) {
+ std::vector<FileStatus> children;
+ Status status = file_system->ListFileStatus(directory, &children);
+ if (status.IsNotExist()) {
+ // Gone since its parent listed it; the rest still stands. Not the root, which
+ // was checked above.
+ continue;
+ }
+ PAIMON_RETURN_NOT_OK(status);
+ for (const FileStatus& child : children) {
+ const std::string name = PathUtil::GetName(child.GetPath());
+ const bool hidden = PartitionPathUtils::IsHiddenName(name);
+ if (child.IsDir()) {
+ const bool is_default_part_dir =
+ exempt_default_part_name && name == options.default_part_name;
+ if (hidden && !is_default_part_dir) {
+ continue;
+ }
+ if (depth == 0 && options.skip_reserved_directories &&
+ IsReservedDirectory(name)) {
+ continue;
+ }
+ // At a partition level, only a directory naming this table's partition is
+ // its data: a caller that deletes what this returns would otherwise remove
+ // files the table never owned. The value-only layout has nothing to check,
+ // since the level alone says which key a directory belongs to.
+ if (children_are_partitions && !options.only_value_in_path &&
+ !options.partition_keys.empty()) {
+ const auto key_index = static_cast<size_t>(depth);
+ if (key_index >= options.partition_keys.size()) {
+ continue;
+ }
+ std::optional<std::pair<std::string, std::string>> key_value =
+ PartitionPathUtils::ExtractPartitionKeyValue(name);
+ if (!key_value || key_value->first != options.partition_keys[key_index]) {
+ continue;
+ }
+ }
+ next.push_back(child.GetPath());
+ } else if (!hidden && !children_are_partitions) {
+ files->emplace_back(child.GetPath(), child.GetLen());
+ }
+ }
+ }
+ level = std::move(next);
+ depth++;
+ }
+ return Status::OK();
+}
+
+} // namespace paimon
diff --git a/src/paimon/core/table/format/format_file_listing.h b/src/paimon/core/table/format/format_file_listing.h
new file mode 100644
index 00000000..0f11c829
--- /dev/null
+++ b/src/paimon/core/table/format/format_file_listing.h
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "paimon/core/table/format/format_data_split.h"
+#include "paimon/status.h"
+
+namespace paimon {
+
+class FileSystem;
+
+/// What the listing has to know about the directory tree below its root.
+struct FormatDataFileListingOptions {
+ /// Zero when the root is already a complete partition.
+ int32_t partition_levels = 0;
+ /// The partition keys of those levels, outermost first, so a directory at a partition level
+ /// can be told from one that is not a partition of this table at all. Empty skips that check,
+ /// which is what a `key=value` layout needs when the caller does not know the keys.
+ std::vector<std::string> partition_keys;
+ /// Whether a partition directory is named by its value alone instead of `key=value`.
+ bool only_value_in_path = false;
+ /// The null partition value's name: in the value-only layout, the one hidden name that is
+ /// table content.
+ std::string default_part_name;
+ /// False when the schema lives in a metastore, where either name is data.
+ bool skip_reserved_directories = false;
+};
+
+/// Finds the data files of a format table in the directory tree it is laid out in.
+class FormatFileListing {
+ public:
+ FormatFileListing() = delete;
+ ~FormatFileListing() = delete;
+
+ /// Whether `name` is metadata this library keeps under a table location. A table whose
+ /// schema lives in a catalog has none.
+ static bool IsReservedDirectory(const std::string& name);
+
+ /// Collects every committed data file under `root`, at any depth.
+ ///
+ /// A hidden `_` / `.` name is skipped and never descended into, since an uncommitted job
+ /// stages output there under ordinary file names; `default_part_name` at a partition level is
+ /// the exception. A missing root is an error - that is not the same as a table with no rows -
+ /// while a directory that disappears mid-listing is skipped.
+ static Status ListDataFiles(const std::shared_ptr<FileSystem>& file_system,
+ const std::string& root,
+ const FormatDataFileListingOptions& options,
+ std::vector<FormatDataSplit::FileMeta>* files);
+};
+
+} // namespace paimon
diff --git a/src/paimon/core/table/format/format_file_listing_test.cpp b/src/paimon/core/table/format/format_file_listing_test.cpp
new file mode 100644
index 00000000..ee4dddee
--- /dev/null
+++ b/src/paimon/core/table/format/format_file_listing_test.cpp
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/core/table/format/format_file_listing.h"
+
+#include <algorithm>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "gtest/gtest.h"
+#include "paimon/common/utils/path_util.h"
+#include "paimon/fs/file_system.h"
+#include "paimon/fs/local/local_file_system.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+
+namespace {
+
+Status WriteAt(const std::shared_ptr<FileSystem>& file_system, const std::string& path) {
+ std::string parent = PathUtil::GetParentDirPath(path);
+ PAIMON_RETURN_NOT_OK(file_system->Mkdirs(parent));
+ return file_system->WriteFile(path, "row\n", /*overwrite=*/true);
+}
+
+/// The listed files as paths relative to `root`, sorted so the order of a listing cannot matter.
+Result<std::vector<std::string>> ListNames(const std::shared_ptr<FileSystem>& file_system,
+ const std::string& root,
+ const FormatDataFileListingOptions& options) {
+ std::vector<FormatDataSplit::FileMeta> files;
+ PAIMON_RETURN_NOT_OK(FormatFileListing::ListDataFiles(file_system, root, options, &files));
+ std::vector<std::string> names;
+ names.reserve(files.size());
+ for (const FormatDataSplit::FileMeta& file : files) {
+ names.push_back(file.file_path.substr(root.size() + 1));
+ }
+ std::sort(names.begin(), names.end());
+ return names;
+}
+
+/// A file system that reports one directory gone when it is listed, standing in for a directory
+/// another job removed between the listing that found it and the listing that descends into it.
+/// Only `ListFileStatus()` is answered that way, since that is where the race lands: the parent
+/// has already handed the name out.
+class VanishingDirectoryFileSystem : public LocalFileSystem {
+ public:
+ explicit VanishingDirectoryFileSystem(std::string vanished) : vanished_(std::move(vanished)) {}
+
+ Status ListFileStatus(const std::string& path,
+ std::vector<FileStatus>* file_status_list) const override {
+ if (path == vanished_) {
+ return Status::NotExist(path + " is gone");
+ }
+ return LocalFileSystem::ListFileStatus(path, file_status_list);
+ }
+
+ private:
+ std::string vanished_;
+};
+
+} // namespace
+
+TEST(FormatFileListingTest, TestDescendsIntoPlainSubdirectories) {
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ std::shared_ptr<FileSystem> file_system = std::make_shared<LocalFileSystem>();
+ // `data-file.path-directory`, and other engines, put data files below the partition directory
+ // rather than directly in it, so stopping at the top level would miss them.
+ ASSERT_OK(WriteAt(file_system, dir->Str() + "/a.parquet"));
+ ASSERT_OK(WriteAt(file_system, dir->Str() + "/nested/b.parquet"));
+ ASSERT_OK(WriteAt(file_system, dir->Str() + "/nested/deeper/c.parquet"));
+
+ ASSERT_OK_AND_ASSIGN(std::vector<std::string> names,
+ ListNames(file_system, dir->Str(), FormatDataFileListingOptions{}));
+ ASSERT_EQ(names, (std::vector<std::string>{"a.parquet", "nested/b.parquet",
+ "nested/deeper/c.parquet"}));
+}
+
+TEST(FormatFileListingTest, TestHiddenNamesAreSkippedAndNotDescendedInto) {
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ std::shared_ptr<FileSystem> file_system = std::make_shared<LocalFileSystem>();
+ // A staging tree holds another job's uncommitted output under ordinary data file names, so
+ // only the directory above them tells the two apart.
+ ASSERT_OK(WriteAt(file_system, dir->Str() + "/a.parquet"));
+ ASSERT_OK(WriteAt(file_system, dir->Str() + "/.b.parquet.tmp"));
+ ASSERT_OK(WriteAt(file_system, dir->Str() + "/_temporary/c.parquet"));
+ ASSERT_OK(WriteAt(file_system, dir->Str() + "/.hive-staging_1/d.parquet"));
+
+ ASSERT_OK_AND_ASSIGN(std::vector<std::string> names,
+ ListNames(file_system, dir->Str(), FormatDataFileListingOptions{}));
+ ASSERT_EQ(names, (std::vector<std::string>{"a.parquet"}));
+}
+
+TEST(FormatFileListingTest, TestDefaultPartitionDirectoryIsTheOneHiddenNameThatIsContent) {
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ std::shared_ptr<FileSystem> file_system = std::make_shared<LocalFileSystem>();
+ // In the value-only layout a partition directory is the bare value, so a null partition is
+ // named `__DEFAULT_PARTITION__` - a hidden name that nonetheless holds table data.
+ ASSERT_OK(WriteAt(file_system, dir->Str() + "/__DEFAULT_PARTITION__/a.parquet"));
+ ASSERT_OK(WriteAt(file_system, dir->Str() + "/_temporary/b.parquet"));
+
+ FormatDataFileListingOptions options;
+ options.partition_levels = 1;
+ options.only_value_in_path = true;
+ options.default_part_name = "__DEFAULT_PARTITION__";
+ ASSERT_OK_AND_ASSIGN(std::vector<std::string> names,
+ ListNames(file_system, dir->Str(), options));
+ ASSERT_EQ(names, (std::vector<std::string>{"__DEFAULT_PARTITION__/a.parquet"}));
+
+ // With no partition level below the root, that name is a staging tree like any other.
+ FormatDataFileListingOptions no_partition_level = options;
+ no_partition_level.partition_levels = 0;
+ ASSERT_OK_AND_ASSIGN(std::vector<std::string> without,
+ ListNames(file_system, dir->Str(), no_partition_level));
+ ASSERT_TRUE(without.empty());
+}
+
+TEST(FormatFileListingTest, TestReservedDirectoriesAreSkippedOnlyWhenTheyAreMetadata) {
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ std::shared_ptr<FileSystem> file_system = std::make_shared<LocalFileSystem>();
+ ASSERT_OK(WriteAt(file_system, dir->Str() + "/a.parquet"));
+ ASSERT_OK(WriteAt(file_system, dir->Str() + "/schema/schema-0"));
+
+ FormatDataFileListingOptions metadata_here;
+ metadata_here.skip_reserved_directories = true;
+ ASSERT_OK_AND_ASSIGN(std::vector<std::string> skipped,
+ ListNames(file_system, dir->Str(), metadata_here));
+ ASSERT_EQ(skipped, (std::vector<std::string>{"a.parquet"}));
+
+ // For a table whose schema lives in a metastore, the location is nothing but data and a
+ // directory of that name is a partition value or a data subdirectory.
+ ASSERT_OK_AND_ASSIGN(std::vector<std::string> kept,
+ ListNames(file_system, dir->Str(), FormatDataFileListingOptions{}));
+ ASSERT_EQ(kept, (std::vector<std::string>{"a.parquet", "schema/schema-0"}));
+}
+
+TEST(FormatFileListingTest, TestMissingRootIsAnErrorButAVanishedSubdirectoryIsNot) {
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ std::shared_ptr<FileSystem> file_system = std::make_shared<LocalFileSystem>();
+ // A root that is not there means the location is wrong or the data is gone. The file systems
+ // here report a missing directory as an empty listing, so this only works because the root is
+ // asked about outright - passing it off as a table with no rows would hide a mistyped path.
+ std::vector<FormatDataSplit::FileMeta> files;
+ ASSERT_NOK(FormatFileListing::ListDataFiles(file_system, dir->Str() + "/absent",
+ FormatDataFileListingOptions{}, &files));
+
+ // A root that is a file, not a directory, is wrong in the same way. The table's own data file
+ // stands in for it, so nothing is left behind to turn up in the listing below.
+ ASSERT_OK(WriteAt(file_system, dir->Str() + "/a.parquet"));
+ ASSERT_NOK_WITH_MSG(FormatFileListing::ListDataFiles(file_system, dir->Str() + "/a.parquet",
+ FormatDataFileListingOptions{}, &files),
+ "is not a directory");
+
+ // A directory below it is another matter: it can be gone by the time the listing reaches it,
+ // and the rest of the listing still stands.
+ const std::string vanished = dir->Str() + "/gone";
+ ASSERT_OK(WriteAt(file_system, vanished + "/b.parquet"));
+ std::shared_ptr<FileSystem> vanishing =
+ std::make_shared<VanishingDirectoryFileSystem>(vanished);
+ ASSERT_OK_AND_ASSIGN(std::vector<std::string> names,
+ ListNames(vanishing, dir->Str(), FormatDataFileListingOptions{}));
+ ASSERT_EQ(names, (std::vector<std::string>{"a.parquet"}));
+
+ // What the same tree lists as when nothing goes missing, so that the listing above is short
+ // because the directory vanished rather than because there was never anything below it.
+ ASSERT_OK_AND_ASSIGN(std::vector<std::string> whole,
+ ListNames(file_system, dir->Str(), FormatDataFileListingOptions{}));
+ ASSERT_EQ(whole, (std::vector<std::string>{"a.parquet", "gone/b.parquet"}));
+}
+
+} // namespace paimon::test
diff --git a/src/paimon/core/table/format/format_file_naming.cpp b/src/paimon/core/table/format/format_file_naming.cpp
new file mode 100644
index 00000000..8442abec
--- /dev/null
+++ b/src/paimon/core/table/format/format_file_naming.cpp
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/core/table/format/format_file_naming.h"
+
+#include <memory>
+
+#include "fmt/format.h"
+#include "paimon/common/utils/string_utils.h"
+#include "paimon/common/utils/uuid.h"
+#include "paimon/core/utils/partition_path_utils.h"
+#include "paimon/defs.h"
+#include "paimon/status.h"
+
+namespace paimon {
+
+namespace {
+
+/// Fails when `value` is anything other than part of a single file name.
+Status CheckFileNameComponent(const std::string& value, const std::string& what) {
+ if (value.find('/') != std::string::npos || value.find('\\') != std::string::npos) {
+ return Status::Invalid(
+ fmt::format("{} '{}' cannot contain a path separator: it names part of one file, not "
+ "a path",
+ what, value));
+ }
+ if (value == "." || value == ".." || value.find("..") != std::string::npos) {
+ return Status::Invalid(fmt::format("{} '{}' cannot contain '..'", what, value));
+ }
+ return Status::OK();
+}
+
+} // namespace
+
+Result<std::unique_ptr<FormatFileNaming>> FormatFileNaming::Create(const std::string& extension,
+ const std::string& prefix) {
+ if (extension.empty()) {
+ return Status::Invalid("format table file naming requires a file extension");
+ }
+ // Both go straight into a file name joined onto a directory, and the file is created before
+ // any commit sees it, so this is the only place a separator or a `..` can be stopped.
+ PAIMON_RETURN_NOT_OK(CheckFileNameComponent(prefix, Options::DATA_FILE_PREFIX));
+ PAIMON_RETURN_NOT_OK(CheckFileNameComponent(extension, "file extension"));
+ // A hidden prefix would name files this table's own scan skips.
+ if (PartitionPathUtils::IsHiddenName(prefix)) {
+ return Status::Invalid(
+ fmt::format("{} '{}' cannot start with '_' or '.': a scan skips every file whose name "
+ "does",
+ Options::DATA_FILE_PREFIX, prefix));
+ }
+ std::string uuid;
+ if (!UUID::Generate(&uuid)) {
+ return Status::Invalid("failed to generate uuid for format table file naming");
+ }
+ return std::unique_ptr<FormatFileNaming>(new FormatFileNaming(uuid, extension, prefix));
+}
+
+std::string FormatFileNaming::NextFileName() {
+ return fmt::format("{}{}-{}.{}", prefix_, uuid_, file_count_++, extension_);
+}
+
+Result<std::string> FormatFileNaming::NextTempFilePath() {
+ // A uuid of its own rather than this write's, as Java Paimon does: `_temporary` is shared,
+ // and a name derived from the target would collide with a retry of the same write.
+ std::string uuid;
+ if (!UUID::Generate(&uuid)) {
+ return Status::Invalid("failed to generate uuid for a staged format table file");
+ }
+ return fmt::format("{}/{}{}", kTempDirName, kTempFilePrefix, uuid);
+}
+
+bool FormatFileNaming::IsTempFilePath(const std::string& relative_path) {
+ const std::string directory_prefix = std::string(kTempDirName) + "/";
+ if (!StringUtils::StartsWith(relative_path, directory_prefix)) {
+ return false;
+ }
+ const std::string name = relative_path.substr(directory_prefix.size());
+ // One level below `_temporary` only: a deeper path is another job's staging tree, not ours.
+ return name.find('/') == std::string::npos && StringUtils::StartsWith(name, kTempFilePrefix) &&
+ name.size() > std::string(kTempFilePrefix).size();
+}
+
+} // namespace paimon
diff --git a/src/paimon/core/table/format/format_file_naming.h b/src/paimon/core/table/format/format_file_naming.h
new file mode 100644
index 00000000..c1d2dc01
--- /dev/null
+++ b/src/paimon/core/table/format/format_file_naming.h
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <memory>
+#include <string>
+
+#include "paimon/result.h"
+
+namespace paimon {
+
+/// Names the data files one format table write produces.
+///
+/// `{prefix}{uuid}-{n}.{extension}`, the convention every paimon writer follows: the uuid belongs
+/// to this write and the counter to its files, so two concurrent writers cannot collide.
+///
+/// A file is staged under `_temporary/.tmp.{uuid}` beside where it will end up and takes its real
+/// name only on commit, as Java Paimon's `RenamingTwoPhaseOutputStream` does. Both the directory
+/// and the name are hidden, the Hive-style convention for uncommitted output that a scan skips.
+///
+/// TODO(SteNicholas): support publishing without a rename. On an object store a rename is a copy
+/// followed by a delete, so a commit pays for the data twice and cannot undo a partly finished
+/// publish - which is why `FormatTableCommit`'s rollback is best-effort. Java's
+/// `MultiPartUploadTwoPhaseOutputStream` writes straight to the final key and defers only
+/// `CompleteMultipartUpload`, making the publish one atomic call with no copy.
+class FormatFileNaming {
+ public:
+ static constexpr char kDefaultDataFilePrefix[] = "data-";
+ /// Directory a staged file waits in, shared with every other writer of the same table.
+ static constexpr char kTempDirName[] = "_temporary";
+ static constexpr char kTempFilePrefix[] = ".tmp.";
+
+ /// @param extension File extension without its dot, which is the format's identifier.
+ /// @param prefix File name prefix, from `data-file.prefix`. It may not be hidden by the
+ /// `_` / `.` convention, since a scan skips every such file.
+ static Result<std::unique_ptr<FormatFileNaming>> Create(const std::string& extension,
+ const std::string& prefix);
+
+ /// The name the next file takes once committed.
+ std::string NextFileName();
+
+ /// Where the next file is staged, relative to the directory it will be published in. Each
+ /// staged name carries a uuid of its own, so two writers sharing `_temporary` cannot collide.
+ Result<std::string> NextTempFilePath();
+
+ /// Whether `relative_path` is one `NextTempFilePath()` could have produced.
+ static bool IsTempFilePath(const std::string& relative_path);
+
+ private:
+ FormatFileNaming(const std::string& uuid, const std::string& extension,
+ const std::string& prefix)
+ : uuid_(uuid), extension_(extension), prefix_(prefix) {}
+
+ std::string uuid_;
+ std::string extension_;
+ std::string prefix_;
+ int64_t file_count_ = 0;
+};
+
+} // namespace paimon
diff --git a/src/paimon/core/table/format/format_file_naming_test.cpp b/src/paimon/core/table/format/format_file_naming_test.cpp
new file mode 100644
index 00000000..55d89060
--- /dev/null
+++ b/src/paimon/core/table/format/format_file_naming_test.cpp
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/core/table/format/format_file_naming.h"
+
+#include <memory>
+#include <string>
+
+#include "gtest/gtest.h"
+#include "paimon/common/utils/string_utils.h"
+#include "paimon/status.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+
+TEST(FormatFileNamingTest, TestNamesAreDataPrefixedAndNumbered) {
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatFileNaming> naming,
+ FormatFileNaming::Create("parquet", FormatFileNaming::kDefaultDataFilePrefix));
+ std::string first = naming->NextFileName();
+ std::string second = naming->NextFileName();
+
+ ASSERT_TRUE(StringUtils::StartsWith(first, "data-"));
+ ASSERT_TRUE(StringUtils::EndsWith(first, "-0.parquet"));
+ ASSERT_TRUE(StringUtils::EndsWith(second, "-1.parquet"));
+ // Both files of one write share its uuid.
+ ASSERT_EQ(first.substr(0, first.size() - std::string("-0.parquet").size()),
+ second.substr(0, second.size() - std::string("-1.parquet").size()));
+}
+
+TEST(FormatFileNamingTest, TestTwoWritesDoNotCollide) {
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatFileNaming> first_write,
+ FormatFileNaming::Create("parquet", FormatFileNaming::kDefaultDataFilePrefix));
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatFileNaming> second_write,
+ FormatFileNaming::Create("parquet", FormatFileNaming::kDefaultDataFilePrefix));
+ ASSERT_NE(first_write->NextFileName(), second_write->NextFileName());
+}
+
+TEST(FormatFileNamingTest, TestTempPathIsAHiddenNameInATemporaryDirectory) {
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatFileNaming> naming,
+ FormatFileNaming::Create("parquet", FormatFileNaming::kDefaultDataFilePrefix));
+ ASSERT_OK_AND_ASSIGN(std::string first, naming->NextTempFilePath());
+ ASSERT_OK_AND_ASSIGN(std::string second, naming->NextTempFilePath());
+ // The `_temporary` directory and the leading '.' are both hidden, which is what a scan skips,
+ // and are the layout Java Paimon stages under.
+ ASSERT_TRUE(StringUtils::StartsWith(first, "_temporary/.tmp.")) << first;
+ ASSERT_TRUE(FormatFileNaming::IsTempFilePath(first));
+ // Each staged name carries a uuid of its own, so two writers staging into the one shared
+ // `_temporary` directory cannot collide.
+ ASSERT_NE(first, second);
+
+ // Anything else is not a path this write staged: a plain file beside the target, a name
+ // outside `_temporary`, or a deeper tree another job staged into.
+ ASSERT_FALSE(FormatFileNaming::IsTempFilePath("data-abc-0.parquet"));
+ ASSERT_FALSE(FormatFileNaming::IsTempFilePath(".data-abc-0.parquet.tmp"));
+ ASSERT_FALSE(FormatFileNaming::IsTempFilePath("_temporary/"));
+ ASSERT_FALSE(FormatFileNaming::IsTempFilePath("_temporary/.tmp."));
+ ASSERT_FALSE(FormatFileNaming::IsTempFilePath("_temporary/attempt_0/part-0.parquet"));
+ ASSERT_FALSE(FormatFileNaming::IsTempFilePath("_temporary/0/.tmp.abc"));
+}
+
+TEST(FormatFileNamingTest, TestPrefixComesFromDataFilePrefix) {
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FormatFileNaming> naming,
+ FormatFileNaming::Create("parquet", "part-"));
+ std::string name = naming->NextFileName();
+ ASSERT_TRUE(StringUtils::StartsWith(name, "part-"));
+ ASSERT_TRUE(StringUtils::EndsWith(name, "-0.parquet"));
+}
+
+TEST(FormatFileNamingTest, TestRejectsAPrefixThatIsNotOneFileNameComponent) {
+ // The prefix goes straight into a file name that is joined onto a directory, and the file is
+ // created before any commit sees it - so a separator or a `..` here would put data outside the
+ // table with nothing left to stop it.
+ for (const char* prefix :
+ {"nested/", "../outside-", "nested/../../outside-", "a\\b", "..", "."}) {
+ ASSERT_NOK(FormatFileNaming::Create("parquet", prefix)) << prefix;
+ }
+ // The extension reaches the same file name, so it is held to the same rule.
+ ASSERT_NOK(FormatFileNaming::Create("../parquet", FormatFileNaming::kDefaultDataFilePrefix));
+}
+
+TEST(FormatFileNamingTest, TestRejectsHiddenPrefix) {
+ // A scan skips every file whose name starts with '_' or '.', so such a prefix would write
+ // rows that can never be read back.
+ ASSERT_NOK(FormatFileNaming::Create("parquet", "_data-"));
+ ASSERT_NOK(FormatFileNaming::Create("parquet", ".data-"));
+}
+
+TEST(FormatFileNamingTest, TestRejectsEmptyExtension) {
+ ASSERT_NOK(FormatFileNaming::Create("", FormatFileNaming::kDefaultDataFilePrefix));
+}
+
+} // namespace paimon::test
diff --git a/src/paimon/core/table/format/format_path_validation.cpp b/src/paimon/core/table/format/format_path_validation.cpp
new file mode 100644
index 00000000..8d8870cd
--- /dev/null
+++ b/src/paimon/core/table/format/format_path_validation.cpp
@@ -0,0 +1,368 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/core/table/format/format_path_validation.h"
+
+#include <cstddef>
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "fmt/format.h"
+#include "paimon/common/utils/path_util.h"
+#include "paimon/common/utils/string_utils.h"
+#include "paimon/core/table/format/format_file_listing.h"
+#include "paimon/core/utils/partition_path_utils.h"
+
+namespace paimon {
+
+namespace {
+
+/// A table location as a prefix of the paths below it.
+struct LocationPrefix {
+ /// As it was written, without its trailing separator: what a path below it is built from,
+ /// so a built path keeps the table's scheme.
+ std::string root;
+ /// A path is compared against these rather than against `root`, so how each was written
+ /// cannot decide whether one is under the other.
+ std::string scheme;
+ std::string authority;
+ /// The location's path inside that file system, without its trailing separator.
+ std::string inner_root;
+ /// Index in a path's own inner path where the first component below the location starts.
+ size_t components_at = 0;
+};
+
+/// A scheme naming the local file system, which a scheme-less path names too.
+bool IsLocalScheme(const std::string& scheme) {
+ return scheme.empty() || StringUtils::EqualsIgnoreCase(scheme, "file");
+}
+
+/// Whether both name the same file system. A scheme is compared ignoring case, as a URI scheme
+/// is, and an absent one reads as `file`, so `/tmp/t` and `file:///tmp/t` are one place.
+bool NamesSameFileSystem(const LocationPrefix& location, const Path& candidate) {
+ if (location.authority != candidate.authority) {
+ return false;
+ }
+ if (IsLocalScheme(location.scheme) && IsLocalScheme(candidate.scheme)) {
+ return true;
+ }
+ return StringUtils::EqualsIgnoreCase(location.scheme, candidate.scheme);
+}
+
+/// `path` split into the file system it names and the path inside it, with repeated separators
+/// collapsed. A `..` is left alone: only the component walk can tell whether it climbs out.
+///
+/// A relative local path is resolved against the working directory, exactly as `LocalFile` does
+/// as it opens one: a table may be located at `./table` while the listing of it comes back under
+/// the absolute path the file system opened, and the two have to compare equal.
+Result<Path> ResolveComparablePath(const std::string& path, const char* subject,
+ const std::string& what) {
+ Result<Path> parsed = PathUtil::ToPath(path);
+ if (!parsed.ok()) {
+ return Status::Invalid(fmt::format("{} cannot be checked: its {} '{}' is not a path: {}",
+ what, subject, path, parsed.status().message()));
+ }
+ Path resolved = std::move(parsed).value();
+ if (!IsLocalScheme(resolved.scheme) || !resolved.authority.empty() ||
+ (!resolved.path.empty() && resolved.path[0] == '/')) {
+ return resolved;
+ }
+ Result<std::string> working_directory = PathUtil::GetWorkingDirectory();
+ if (!working_directory.ok()) {
+ return Status::Invalid(
+ fmt::format("{} cannot be checked: its {} '{}' is relative to the working directory, "
+ "which cannot be read: {}",
+ what, subject, path, working_directory.status().message()));
+ }
+ // Joined the way `LocalFile` joins it, so a `.` component the file system keeps in the paths
+ // it hands back is kept here too and the two still compare equal.
+ resolved.path = PathUtil::JoinPath(working_directory.value(), resolved.path);
+ return resolved;
+}
+
+/// Resolves `directory` into the prefix the paths below it share, so that no two checks here can
+/// disagree about what "under the table location" means.
+///
+/// A location of nothing but separators, or an authority with no path, is its file system's root,
+/// which is its own separator: what follows starts one character in, not two. An empty location
+/// is refused, since as a prefix it would pass every absolute path.
+Result<LocationPrefix> ResolveLocationPrefix(const std::string& directory, const char* subject,
+ const std::string& what) {
+ if (directory.empty()) {
+ return Status::Invalid(fmt::format(
+ "{} cannot be checked: its {} is empty, and an empty path is a prefix of nothing", what,
+ subject));
+ }
+ PAIMON_ASSIGN_OR_RAISE(Path parsed, ResolveComparablePath(directory, subject, what));
+
+ LocationPrefix prefix;
+ prefix.scheme = std::move(parsed.scheme);
+ prefix.authority = std::move(parsed.authority);
+ size_t root_end = directory.size();
+ while (root_end > 0 && directory[root_end - 1] == '/') {
+ root_end--;
+ }
+ size_t inner_end = parsed.path.size();
+ while (inner_end > 0 && parsed.path[inner_end - 1] == '/') {
+ inner_end--;
+ }
+ if (inner_end == 0) {
+ prefix.root = root_end == 0 ? "/" : directory.substr(0, root_end);
+ prefix.inner_root = "/";
+ prefix.components_at = 1;
+ return prefix;
+ }
+ prefix.root = directory.substr(0, root_end);
+ prefix.inner_root = parsed.path.substr(0, inner_end);
+ prefix.components_at = inner_end + 1;
+ return prefix;
+}
+
+/// Whether `candidate` names something below `prefix`, on the same file system and by whole path
+/// components.
+bool IsUnderPrefix(const Path& candidate, const LocationPrefix& prefix) {
+ if (!NamesSameFileSystem(prefix, candidate)) {
+ return false;
+ }
+ if (candidate.path.size() <= prefix.components_at ||
+ candidate.path.compare(0, prefix.inner_root.size(), prefix.inner_root) != 0) {
+ return false;
+ }
+ // The file system root is its own separator; every other location is followed by one.
+ return prefix.components_at == prefix.inner_root.size() ||
+ candidate.path[prefix.inner_root.size()] == '/';
+}
+
+} // namespace
+
+Status FormatPathValidation::ValidatePathUnderLocation(const std::string& path,
+ const std::string& location,
+ const std::string& what) {
+ PAIMON_ASSIGN_OR_RAISE(LocationPrefix prefix,
+ ResolveLocationPrefix(location, "table location", what));
+ PAIMON_ASSIGN_OR_RAISE(Path candidate, ResolveComparablePath(path, "path", what));
+ if (!IsUnderPrefix(candidate, prefix)) {
+ return Status::Invalid(fmt::format(
+ "{} names '{}', which is not under the table location '{}'", what, path, location));
+ }
+
+ // `<table>/../victim` passes any prefix test and still resolves outside the table.
+ size_t begin = prefix.components_at;
+ while (begin <= candidate.path.size()) {
+ size_t end = candidate.path.find('/', begin);
+ if (end == std::string::npos) {
+ end = candidate.path.size();
+ }
+ const std::string component = candidate.path.substr(begin, end - begin);
+ if (component.empty() || component == "." || component == "..") {
+ return Status::Invalid(fmt::format(
+ "{} names '{}', whose path does not stay inside the table location", what, path));
+ }
+ begin = end + 1;
+ }
+ return Status::OK();
+}
+
+namespace {
+
+/// Fails when a scan would not reach `path`, whose last component names a file when `ends_in_file`
+/// and a directory otherwise. One walk serves both: the distinction matters only for the last
+/// component, which as a directory may be reserved or stand for a null partition.
+Status ValidateComponentsAreVisible(const std::shared_ptr<FormatTable>& table,
+ const std::string& path, bool ends_in_file,
+ const std::string& what) {
+ const std::vector<std::string>& partition_keys = table->PartitionKeys();
+ PAIMON_ASSIGN_OR_RAISE(LocationPrefix prefix,
+ ResolveLocationPrefix(table->Location(), "table location", what));
+ // Both sides as their file system holds them, so a `file:` URI checked against a bare local
+ // location does not start the walk part way through the location itself.
+ PAIMON_ASSIGN_OR_RAISE(Path candidate, ResolveComparablePath(path, "path", what));
+ const bool only_value = table->PartitionOnlyValueInPath();
+
+ size_t begin = prefix.components_at;
+ size_t level = 0;
+ while (begin <= candidate.path.size()) {
+ size_t end = candidate.path.find('/', begin);
+ const bool is_last = end == std::string::npos;
+ if (is_last) {
+ end = candidate.path.size();
+ }
+ const std::string component = candidate.path.substr(begin, end - begin);
+ const bool is_directory = !is_last || !ends_in_file;
+
+ // The one hidden name a scan reads, and only where a partition directory belongs.
+ const bool is_default_partition_dir = is_directory && only_value &&
+ level < partition_keys.size() &&
+ component == table->PartitionDefaultName();
+ if (PartitionPathUtils::IsHiddenName(component) && !is_default_partition_dir) {
+ return Status::Invalid(fmt::format(
+ "{} names '{}', which a scan of this table would skip: '{}' is hidden, and that is "
+ "how an uncommitted job marks its output",
+ what, path, component));
+ }
+ // Only right below the location, and only when the schema lives there. A value-only
+ // partition lands here unescaped, so one named `schema` would be written over it.
+ if (level == 0 && is_directory && table->LocationCarriesPaimonMetadata() &&
+ FormatFileListing::IsReservedDirectory(component)) {
+ return Status::Invalid(fmt::format(
+ "{} names '{}', where '{}' is this table's own metadata rather than data", what,
+ path, component));
+ }
+ begin = end + 1;
+ level++;
+ }
+ return Status::OK();
+}
+
+} // namespace
+
+Status FormatPathValidation::ValidateFileIsVisible(const std::shared_ptr<FormatTable>& table,
+ const std::string& file_path,
+ const std::string& what) {
+ return ValidateComponentsAreVisible(table, file_path, /*ends_in_file=*/true, what);
+}
+
+Result<bool> FormatPathValidation::IsTableLocation(const std::shared_ptr<FormatTable>& table,
+ const std::string& directory) {
+ const std::string what = fmt::format("table {}", table->FullName());
+ PAIMON_ASSIGN_OR_RAISE(LocationPrefix location,
+ ResolveLocationPrefix(table->Location(), "table location", what));
+ PAIMON_ASSIGN_OR_RAISE(LocationPrefix candidate,
+ ResolveLocationPrefix(directory, "directory", what));
+ return NamesSameFileSystem(location,
+ Path(candidate.scheme, candidate.authority, candidate.inner_root)) &&
+ location.inner_root == candidate.inner_root;
+}
+
+Status FormatPathValidation::ValidateDirectoryIsVisible(const std::shared_ptr<FormatTable>& table,
+ const std::string& directory,
+ const std::string& what) {
+ // Trailing separators go first, or the walk below sees an empty last component.
+ PAIMON_ASSIGN_OR_RAISE(LocationPrefix directory_prefix,
+ ResolveLocationPrefix(directory, "directory", what));
+ return ValidateComponentsAreVisible(table, directory_prefix.root, /*ends_in_file=*/false, what);
+}
+
+Status FormatPathValidation::ValidatePartitionKeys(
+ const std::shared_ptr<FormatTable>& table, const std::map<std::string, std::string>& partition,
+ const std::string& what) {
+ const std::vector<std::string>& partition_keys = table->PartitionKeys();
+ if (partition.size() != partition_keys.size()) {
+ return Status::Invalid(
+ fmt::format("{} carries {} partition values but table {} is partitioned by {} fields",
+ what, partition.size(), table->FullName(), partition_keys.size()));
+ }
+ for (const std::string& partition_key : partition_keys) {
+ if (partition.find(partition_key) == partition.end()) {
+ return Status::Invalid(fmt::format("{} does not carry a value for partition field '{}'",
+ what, partition_key));
+ }
+ }
+ return Status::OK();
+}
+
+Status FormatPathValidation::ValidateFileInPartition(
+ const std::shared_ptr<FormatTable>& table, const std::string& file_path,
+ const std::map<std::string, std::string>& partition, const std::string& what) {
+ const std::vector<std::string>& partition_keys = table->PartitionKeys();
+ PAIMON_ASSIGN_OR_RAISE(LocationPrefix prefix,
+ ResolveLocationPrefix(table->Location(), "table location", what));
+ PAIMON_ASSIGN_OR_RAISE(Path candidate, ResolveComparablePath(file_path, "path", what));
+ // The directory components between the location and the file name; the leading
+ // `partition_keys.size()` of them are the partition directories.
+ std::vector<std::string> components;
+ size_t begin = prefix.components_at;
+ while (begin < candidate.path.size()) {
+ size_t end = candidate.path.find('/', begin);
+ if (end == std::string::npos) {
+ break;
+ }
+ components.push_back(candidate.path.substr(begin, end - begin));
+ begin = end + 1;
+ }
+ if (components.size() < partition_keys.size()) {
+ return Status::Invalid(
+ fmt::format("{} names '{}', which sits above the {} partition directories of table {}",
+ what, file_path, partition_keys.size(), table->FullName()));
+ }
+
+ const bool only_value = table->PartitionOnlyValueInPath();
+ for (size_t i = 0; i < partition_keys.size(); i++) {
+ const std::string& partition_key = partition_keys[i];
+ std::string value;
+ if (only_value) {
+ value = PartitionPathUtils::UnescapePathName(components[i]);
+ } else {
+ std::optional<std::pair<std::string, std::string>> key_value =
+ PartitionPathUtils::ExtractPartitionKeyValue(components[i]);
+ if (!key_value || key_value->first != partition_key) {
+ return Status::Invalid(
+ fmt::format("{} names '{}', whose directory '{}' is not a partition of '{}'",
+ what, file_path, components[i], partition_key));
+ }
+ value = key_value->second;
+ }
+ auto iter = partition.find(partition_key);
+ if (iter == partition.end() || iter->second != value) {
+ return Status::Invalid(fmt::format(
+ "{} sits in the '{}' partition of '{}' but claims '{}'", what, value, partition_key,
+ iter == partition.end() ? std::string("nothing") : iter->second));
+ }
+ }
+ return Status::OK();
+}
+
+Result<std::string> FormatPathValidation::BuildPartitionDirectory(
+ const std::shared_ptr<FormatTable>& table,
+ const std::map<std::string, std::string>& partition) {
+ const std::vector<std::string>& partition_keys = table->PartitionKeys();
+ std::vector<std::pair<std::string, std::string>> ordered_partition;
+ ordered_partition.reserve(partition_keys.size());
+ for (const std::string& partition_key : partition_keys) {
+ auto iter = partition.find(partition_key);
+ if (iter == partition.end()) {
+ return Status::Invalid(fmt::format("no value for partition field '{}' of table {}",
+ partition_key, table->FullName()));
+ }
+ ordered_partition.emplace_back(partition_key, iter->second);
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::string partition_path,
+ PartitionPathUtils::GeneratePartitionPath(
+ ordered_partition, table->PartitionOnlyValueInPath()));
+ PAIMON_ASSIGN_OR_RAISE(LocationPrefix prefix,
+ ResolveLocationPrefix(table->Location(), "table location",
+ fmt::format("table {}", table->FullName())));
+ if (partition_path.empty()) {
+ return prefix.root;
+ }
+ const std::string directory = PathUtil::JoinPath(prefix.root, partition_path);
+ // A directory a scan would skip could be written but never read back, and an overwrite of
+ // it would clear whatever does live there.
+ std::string described;
+ for (const auto& [key, value] : ordered_partition) {
+ described += described.empty() ? "" : ", ";
+ described += fmt::format("{}={}", key, value);
+ }
+ PAIMON_RETURN_NOT_OK(FormatPathValidation::ValidateDirectoryIsVisible(
+ table, directory, fmt::format("partition {}", described)));
+ return directory;
+}
+
+} // namespace paimon
diff --git a/src/paimon/core/table/format/format_path_validation.h b/src/paimon/core/table/format/format_path_validation.h
new file mode 100644
index 00000000..7574dfaa
--- /dev/null
+++ b/src/paimon/core/table/format/format_path_validation.h
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <map>
+#include <memory>
+#include <string>
+
+#include "paimon/result.h"
+#include "paimon/status.h"
+#include "paimon/table/format/format_table.h"
+
+namespace paimon {
+
+/// Checks that a path a caller handed over really names something of this format table.
+///
+/// A split and a commit message arrive through an interface taking the base type, so what comes
+/// in may belong to another plan or another table, and nothing further down re-checks it.
+class FormatPathValidation {
+ public:
+ FormatPathValidation() = delete;
+ ~FormatPathValidation() = delete;
+
+ /// Fails when `path` is not a file inside `location`. By path component, not by string
+ /// prefix: `<location>/../victim` starts with the location and still resolves outside it.
+ ///
+ /// Both are resolved into the file system they name and the path inside it first, so
+ /// `/tmp/table` holds `file:///tmp/table/data.parquet`; any other scheme, and the authority,
+ /// must match. Only the text is checked, so a symlink out of the table is not caught.
+ static Status ValidatePathUnderLocation(const std::string& path, const std::string& location,
+ const std::string& what);
+
+ /// The directory a partition's files belong in. Only for building a path to write or clear;
+ /// to check one that already exists use `ValidateFileInPartition()`, since another engine may
+ /// spell the same value differently.
+ static Result<std::string> BuildPartitionDirectory(
+ const std::shared_ptr<FormatTable>& table,
+ const std::map<std::string, std::string>& partition);
+
+ /// Fails when `file_path` does not sit in the directory `partition` names, which would
+ /// publish rows under a partition they never had. Compared on the values the names spell out,
+ /// since `100%` and `100%25` are one value; levels below the partition are allowed.
+ static Status ValidateFileInPartition(const std::shared_ptr<FormatTable>& table,
+ const std::string& file_path,
+ const std::map<std::string, std::string>& partition,
+ const std::string& what);
+
+ /// Fails when `file_path` is one a scan would never return: a hidden `_` / `.` name, or this
+ /// table's own `schema` or `branch`. A split from elsewhere never went through that listing.
+ static Status ValidateFileIsVisible(const std::shared_ptr<FormatTable>& table,
+ const std::string& file_path, const std::string& what);
+
+ /// Whether `directory` is the table's own location, however either was written. It decides
+ /// whether `schema` and `branch` below it are metadata, so a trailing separator or a `file:`
+ /// scheme on one side alone must not make them differ.
+ static Result<bool> IsTableLocation(const std::shared_ptr<FormatTable>& table,
+ const std::string& directory);
+
+ /// `ValidateFileIsVisible()` for a path ending in a directory: in the value-only layout a
+ /// partition named `schema` lands where a file system catalog keeps the table's metadata.
+ static Status ValidateDirectoryIsVisible(const std::shared_ptr<FormatTable>& table,
+ const std::string& directory, const std::string& what);
+
+ /// Fails when `partition` does not name exactly the table's partition keys.
+ static Status ValidatePartitionKeys(const std::shared_ptr<FormatTable>& table,
+ const std::map<std::string, std::string>& partition,
+ const std::string& what);
+};
+
+} // namespace paimon
diff --git a/src/paimon/core/table/format/format_table.cpp b/src/paimon/core/table/format/format_table.cpp
new file mode 100644
index 00000000..a039baa4
--- /dev/null
+++ b/src/paimon/core/table/format/format_table.cpp
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/table/format/format_table.h"
+
+#include <optional>
+
+#include "fmt/format.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/common/utils/options_utils.h"
+#include "paimon/common/utils/string_utils.h"
+#include "paimon/core/core_options.h"
+#include "paimon/core/options/table_type.h"
+#include "paimon/core/schema/schema_manager.h"
+#include "paimon/core/schema/schema_validation.h"
+#include "paimon/core/schema/table_schema.h"
+#include "paimon/defs.h"
+#include "paimon/fs/file_system.h"
+
+namespace paimon {
+
+namespace {
+constexpr char kDefaultFileFormat[] = "parquet";
+} // namespace
+
+FormatTable::FormatTable(const std::shared_ptr<FileSystem>& file_system,
+ const std::string& location, const Identifier& identifier,
+ const std::shared_ptr<DataSchema>& schema,
+ const std::map<std::string, std::string>& options, Format format,
+ const std::string& file_compression,
+ const std::string& partition_default_name,
+ bool partition_only_value_in_path, bool location_carries_paimon_metadata)
+ : file_system_(file_system),
+ location_(location),
+ identifier_(identifier),
+ schema_(schema),
+ options_(options),
+ format_(format),
+ file_compression_(file_compression),
+ partition_default_name_(partition_default_name),
+ partition_only_value_in_path_(partition_only_value_in_path),
+ location_carries_paimon_metadata_(location_carries_paimon_metadata) {}
+
+FormatTable::~FormatTable() = default;
+
+Result<FormatTable::Format> FormatTable::ParseFormat(const std::string& file_format) {
+ std::string normalized = StringUtils::ToLowerCase(file_format);
+ if (normalized == "parquet") {
+ return Format::PARQUET;
+ }
+ if (normalized == "orc") {
+ return Format::ORC;
+ }
+ if (normalized == "csv" || normalized == "text" || normalized == "json" ||
+ normalized == "mosaic") {
+ return Status::NotImplemented(
+ fmt::format("format table file format '{}' is not supported by paimon-cpp yet. "
+ "Supported formats: parquet, orc",
+ file_format));
+ }
+ return Status::Invalid(fmt::format(
+ "format table unsupported file format: {}. Supported formats: parquet, orc", file_format));
+}
+
+std::string FormatTable::FormatToString(Format format) {
+ switch (format) {
+ case Format::PARQUET:
+ return "parquet";
+ case Format::ORC:
+ return "orc";
+ }
+ return "unknown";
+}
+
+Result<std::shared_ptr<FormatTable>> FormatTable::Create(
+ const std::shared_ptr<FileSystem>& file_system, const std::string& table_path,
+ const Identifier& identifier, const std::map<std::string, std::string>& dynamic_options) {
+ if (file_system == nullptr) {
+ return Status::Invalid("format table requires a file system");
+ }
+ if (table_path.empty()) {
+ return Status::Invalid(
+ fmt::format("format table {} requires a location", identifier.GetFullName()));
+ }
+ PAIMON_ASSIGN_OR_RAISE(bool exist, file_system->Exists(table_path));
+ if (!exist) {
+ return Status::NotExist(fmt::format("{} not exist", identifier.ToString()));
+ }
+ SchemaManager schema_manager(file_system, table_path);
+ PAIMON_ASSIGN_OR_RAISE(std::optional<std::shared_ptr<TableSchema>> latest_schema,
+ schema_manager.Latest());
+ if (!latest_schema) {
+ return Status::NotExist(
+ fmt::format("load table schema for {} failed", identifier.ToString()));
+ }
+ // The schema was just read from under the table path, so that is where the metadata lives.
+ return Create(file_system, table_path, identifier,
+ checked_pointer_cast<DataSchema>(*latest_schema),
+ /*location_carries_paimon_metadata=*/true, dynamic_options);
+}
+
+Result<std::shared_ptr<FormatTable>> FormatTable::Create(
+ const std::shared_ptr<FileSystem>& file_system, const std::string& location,
+ const Identifier& identifier, const std::shared_ptr<DataSchema>& schema,
+ bool location_carries_paimon_metadata,
+ const std::map<std::string, std::string>& dynamic_options) {
+ if (file_system == nullptr) {
+ return Status::Invalid("format table requires a file system");
+ }
+ if (location.empty()) {
+ // Every path is checked against the location, and an empty one is a prefix of nothing.
+ return Status::Invalid(
+ fmt::format("format table {} requires a location", identifier.GetFullName()));
+ }
+ if (schema == nullptr) {
+ return Status::Invalid("format table requires a schema");
+ }
+ // From the schema alone: `type` is structural, so an option given at the call must not let
+ // one read open a managed table as a format table, or hide one.
+ PAIMON_ASSIGN_OR_RAISE(bool is_format_table, TableTypeDefine::IsFormatTable(schema->Options()));
+ if (!is_format_table) {
+ return Status::Invalid(
+ fmt::format("table {} is not a format table, its '{}' option is not "
+ "'{}'",
+ identifier.GetFullName(), Options::TYPE, TableTypeDefine::kFormatTable));
+ }
+ // What the schema stored with anything given at the call on top, the precedence every
+ // context builder promises. `type` was read above and is not one of them.
+ std::map<std::string, std::string> options = schema->Options();
+ for (const auto& [key, value] : dynamic_options) {
+ if (key == Options::TYPE) {
+ continue;
+ }
+ options[key] = value;
+ }
+
+ // The checks table creation would have run, again: this schema may have come from a rest
+ // catalog or straight from a caller. Run against the merged options, so that an option this
+ // refuses is refused wherever it came from.
+ Status valid = SchemaValidation::ValidateGenericDataSchema(*schema);
+ if (valid.ok()) {
+ valid = SchemaValidation::ValidateFormatTableSchema(*schema, options, file_system);
+ }
+ if (!valid.ok()) {
+ return Status(valid.code(), fmt::format("cannot open format table {}: {}",
+ identifier.GetFullName(), valid.message()));
+ }
+
+ // Before `CoreOptions`, which resolves `file.format` through the factories and would fail with
+ // a missing-factory error where this reports the format by name.
+ PAIMON_ASSIGN_OR_RAISE(std::string file_format,
+ OptionsUtils::GetValueFromMap<std::string>(options, Options::FILE_FORMAT,
+ kDefaultFileFormat));
+ PAIMON_ASSIGN_OR_RAISE(Format format, ParseFormat(file_format));
+
+ // Everything else through `CoreOptions`, so an option means what it does elsewhere.
+ PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options, file_system));
+
+ return std::shared_ptr<FormatTable>(new FormatTable(
+ file_system, location, identifier, schema, options, format,
+ core_options.FormatTableFileCompression(), core_options.GetPartitionDefaultName(),
+ core_options.FormatTablePartitionOnlyValueInPath(), location_carries_paimon_metadata));
+}
+
+Result<std::shared_ptr<FormatTable>> FormatTable::Copy(
+ const std::shared_ptr<FormatTable>& table,
+ const std::map<std::string, std::string>& dynamic_options) {
+ if (table == nullptr) {
+ return Status::Invalid("format table requires a table to copy");
+ }
+ if (dynamic_options.empty()) {
+ return table;
+ }
+ // The options this table already carries go back in as dynamic ones, so an option an earlier
+ // call set is not lost to the schema's own value underneath it.
+ std::map<std::string, std::string> options = table->options_;
+ for (const auto& [key, value] : dynamic_options) {
+ options[key] = value;
+ }
+ return Create(table->file_system_, table->location_, table->identifier_, table->schema_,
+ table->location_carries_paimon_metadata_, options);
+}
+
+const std::vector<std::string>& FormatTable::PartitionKeys() const {
+ return schema_->PartitionKeys();
+}
+
+std::string FormatTable::FullName() const {
+ return identifier_.GetFullName();
+}
+
+Result<std::unique_ptr<::ArrowSchema>> FormatTable::GetArrowSchema() const {
+ return schema_->GetArrowSchema();
+}
+
+} // namespace paimon
diff --git a/src/paimon/core/table/format/format_table_commit.cpp b/src/paimon/core/table/format/format_table_commit.cpp
new file mode 100644
index 00000000..9a3abc16
--- /dev/null
+++ b/src/paimon/core/table/format/format_table_commit.cpp
@@ -0,0 +1,390 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/core/table/format/format_table_commit.h"
+
+#include <algorithm>
+#include <cstddef>
+#include <map>
+#include <set>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "fmt/format.h"
+#include "fmt/ranges.h"
+#include "paimon/common/utils/path_util.h"
+#include "paimon/common/utils/string_utils.h"
+#include "paimon/core/core_options.h"
+#include "paimon/core/table/format/format_file_listing.h"
+#include "paimon/core/table/format/format_file_naming.h"
+#include "paimon/core/table/format/format_path_validation.h"
+#include "paimon/core/utils/duration.h"
+#include "paimon/core/utils/partition_path_utils.h"
+#include "paimon/fs/file_system.h"
+#include "paimon/logging.h"
+
+namespace paimon {
+
+namespace {
+
+Logger* CommitLogger() {
+ static std::unique_ptr<Logger> logger = Logger::GetLogger("FormatTableCommit");
+ return logger.get();
+}
+
+/// Fails when a commit message does not describe a file of this table. Only the shape of a
+/// message is checked, not who produced it.
+Status ValidateCommitMessage(const FormatCommitMessage& message,
+ const std::shared_ptr<FormatTable>& table,
+ const std::map<std::string, std::string>& static_partition) {
+ const std::string what = fmt::format("commit message {}", message.ToString());
+ PAIMON_RETURN_NOT_OK(FormatPathValidation::ValidatePathUnderLocation(message.file_path,
+ table->Location(), what));
+ PAIMON_RETURN_NOT_OK(FormatPathValidation::ValidatePathUnderLocation(message.temp_file_path,
+ table->Location(), what));
+
+ // `<publish directory>/_temporary/.tmp.<uuid>`, which a scan skips and which is what Java
+ // Paimon's `RenamingTwoPhaseOutputStream` stages under.
+ const std::string publish_directory = PathUtil::GetParentDirPath(message.file_path);
+ const std::string directory_prefix = publish_directory + "/";
+ if (!StringUtils::StartsWith(message.temp_file_path, directory_prefix) ||
+ !FormatFileNaming::IsTempFilePath(message.temp_file_path.substr(directory_prefix.size()))) {
+ return Status::Invalid(fmt::format(
+ "{} does not stage its file under '{}/{}...' beside where it will be published, so it "
+ "may already be visible or belong to another directory",
+ what, FormatFileNaming::kTempDirName, FormatFileNaming::kTempFilePrefix));
+ }
+ // Otherwise an overwrite could clear the old data and publish a file nothing can ever read.
+ PAIMON_RETURN_NOT_OK(
+ FormatPathValidation::ValidateFileIsVisible(table, message.file_path, what));
+ if (message.record_count < 0 || message.file_size < 0) {
+ return Status::Invalid(fmt::format("{} reports a negative row count or file size", what));
+ }
+
+ // Otherwise an overwrite would clear the wrong partition.
+ PAIMON_RETURN_NOT_OK(
+ FormatPathValidation::ValidatePartitionKeys(table, message.partition, what));
+ PAIMON_RETURN_NOT_OK(FormatPathValidation::ValidateFileInPartition(table, message.file_path,
+ message.partition, what));
+
+ // Otherwise the file would be published into a partition this commit never cleared.
+ for (const auto& [key, value] : static_partition) {
+ auto iter = message.partition.find(key);
+ if (iter == message.partition.end() || iter->second != value) {
+ return Status::Invalid(fmt::format(
+ "{} is not in the static partition '{}={}' this commit writes", what, key, value));
+ }
+ }
+ return Status::OK();
+}
+
+/// Fails when `static_partition` cannot name a directory of this table. The keys must be a prefix
+/// of the partition keys, since a partition directory nests below the one before it.
+Status ValidateStaticPartition(const std::map<std::string, std::string>& static_partition,
+ const std::vector<std::string>& partition_keys,
+ const std::string& table_name) {
+ if (static_partition.empty()) {
+ return Status::OK();
+ }
+ if (partition_keys.empty()) {
+ return Status::Invalid(fmt::format(
+ "format table {} is not partitioned, so a static partition names nothing", table_name));
+ }
+ for (const auto& entry : static_partition) {
+ const std::string& key = entry.first;
+ if (std::find(partition_keys.begin(), partition_keys.end(), key) == partition_keys.end()) {
+ return Status::Invalid(
+ fmt::format("'{}' is not a partition key of format table {}", key, table_name));
+ }
+ }
+ bool missing_leading_key = false;
+ for (const std::string& partition_key : partition_keys) {
+ const bool named = static_partition.find(partition_key) != static_partition.end();
+ if (named && missing_leading_key) {
+ return Status::Invalid(
+ fmt::format("static partition column '{}' of format table {} cannot be given "
+ "without the partition columns it nests under",
+ partition_key, table_name));
+ }
+ if (!named) {
+ missing_leading_key = true;
+ }
+ }
+ return Status::OK();
+}
+
+} // namespace
+
+FormatTableCommit::FormatTableCommit(const std::shared_ptr<FormatTable>& table, bool overwrite,
+ const std::map<std::string, std::string>& static_partition,
+ bool dynamic_partition_overwrite)
+ : table_(table),
+ overwrite_(overwrite),
+ static_partition_(static_partition),
+ dynamic_partition_overwrite_(dynamic_partition_overwrite) {}
+
+FormatTableCommit::~FormatTableCommit() = default;
+
+Result<std::unique_ptr<FormatTableCommit>> FormatTableCommit::Create(
+ const std::shared_ptr<FormatTable>& table, bool overwrite,
+ const std::map<std::string, std::string>& static_partition) {
+ if (table == nullptr) {
+ return Status::Invalid("format table commit requires a table");
+ }
+ PAIMON_RETURN_NOT_OK(
+ ValidateStaticPartition(static_partition, table->PartitionKeys(), table->FullName()));
+ // Through `CoreOptions`, so an option given at the call reaches this as it does the read.
+ PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options,
+ CoreOptions::FromMap(table->Options(), table->GetFileSystem()));
+ return std::unique_ptr<FormatTableCommit>(new FormatTableCommit(
+ table, overwrite, static_partition, core_options.DynamicPartitionOverwrite()));
+}
+
+bool FormatTableCommit::ReplacesOnlyWrittenPartitions() const {
+ return !table_->PartitionKeys().empty() && dynamic_partition_overwrite_;
+}
+
+Status FormatTableCommit::DeletePreviousDataFiles(const std::string& directory,
+ int32_t partition_levels) const {
+ std::shared_ptr<FileSystem> file_system = table_->GetFileSystem();
+ FormatDataFileListingOptions listing;
+ listing.partition_levels = partition_levels;
+ // The keys of the levels still below `directory`: a static partition naming the leading keys
+ // leaves the trailing ones. Without them the listing would descend any directory it found,
+ // and this deletes what it lists.
+ const std::vector<std::string>& partition_keys = table_->PartitionKeys();
+ if (partition_levels > 0 && static_cast<size_t>(partition_levels) <= partition_keys.size()) {
+ listing.partition_keys.assign(partition_keys.end() - partition_levels,
+ partition_keys.end());
+ }
+ listing.only_value_in_path = table_->PartitionOnlyValueInPath();
+ listing.default_part_name = table_->PartitionDefaultName();
+ // Only right at the location are `schema` and `branch` metadata; below it they are data.
+ PAIMON_ASSIGN_OR_RAISE(bool at_location,
+ FormatPathValidation::IsTableLocation(table_, directory));
+ listing.skip_reserved_directories = at_location && table_->LocationCarriesPaimonMetadata();
+ std::vector<FormatDataSplit::FileMeta> files;
+ // Committed data files only: a staging directory holds another writer's uncommitted output.
+ Status listed = FormatFileListing::ListDataFiles(file_system, directory, listing, &files);
+ // A format table's directory need not exist yet: an external table may be loaded from a schema
+ // alone, and the first write to it may be an overwrite. Nothing is there to replace, which is
+ // what Java's `previousDataFiles()` makes of a missing directory too.
+ if (listed.IsNotExist()) {
+ return Status::OK();
+ }
+ PAIMON_RETURN_NOT_OK(listed);
+ for (const FormatDataSplit::FileMeta& file : files) {
+ Status status = file_system->Delete(file.file_path, /*recursive=*/false);
+ if (!status.ok() && !status.IsNotExist()) {
+ return status;
+ }
+ }
+ return Status::OK();
+}
+
+Status FormatTableCommit::Commit(const std::vector<FormatCommitMessage>& commit_messages) {
+ Status status = CommitImpl(commit_messages);
+ if (!status.ok()) {
+ // The write has already prepared its commit, so nothing else will clean up what is staged.
+ Status abort_status = Abort(commit_messages);
+ if (!abort_status.ok()) {
+ PAIMON_LOG_WARN(CommitLogger(), "Failed to clean up table %s after a failed commit: %s",
+ table_->FullName().c_str(), abort_status.ToString().c_str());
+ }
+ }
+ return status;
+}
+
+Status FormatTableCommit::CommitImpl(const std::vector<FormatCommitMessage>& commit_messages) {
+ std::shared_ptr<FileSystem> file_system = table_->GetFileSystem();
+ const std::vector<std::string>& partition_keys = table_->PartitionKeys();
+ Duration duration;
+ PAIMON_LOG_INFO(CommitLogger(), "Ready to %s %zu messages to format table %s",
+ overwrite_ ? "overwrite with" : "commit", commit_messages.size(),
+ table_->FullName().c_str());
+
+ for (const FormatCommitMessage& message : commit_messages) {
+ PAIMON_RETURN_NOT_OK(ValidateCommitMessage(message, table_, static_partition_));
+ }
+
+ // Checked against disk before anything moves: an overwrite clears the old data first, so a
+ // staged file that turns out to be missing would lose the old rows without gaining the new.
+ std::set<std::string> targets;
+ for (const FormatCommitMessage& message : commit_messages) {
+ if (!targets.insert(message.file_path).second) {
+ return Status::Invalid(fmt::format(
+ "two commit messages would publish {}, so one would overwrite the other",
+ message.file_path));
+ }
+ Result<FileStatus> staged = file_system->GetFileStatus(message.temp_file_path);
+ if (!staged.ok()) {
+ // `Invalid` whatever the file system said, since the fault is the message.
+ return Status::Invalid(fmt::format("the staged file {} cannot be read: {}",
+ message.temp_file_path, staged.status().ToString()));
+ }
+ // `rename` moves a directory as readily as a file.
+ if (staged.value().IsDir()) {
+ return Status::Invalid(fmt::format("the staged path {} is a directory, not a file",
+ message.temp_file_path));
+ }
+ if (message.file_size != staged.value().GetLen()) {
+ return Status::Invalid(
+ fmt::format("the staged file {} is {} bytes but the commit message says {}",
+ message.temp_file_path, staged.value().GetLen(), message.file_size));
+ }
+ }
+
+ // What an overwrite replaces is cleared first: it removes committed files only, and this
+ // commit's own are still hidden, so neither can take the other out.
+ std::vector<std::string> replaced_directories;
+ if (!static_partition_.empty()) {
+ // The spec names the leading keys in order, so the path may be a prefix with the
+ // partitions of the unnamed keys below it.
+ std::vector<std::pair<std::string, std::string>> ordered_partition;
+ ordered_partition.reserve(static_partition_.size());
+ for (const std::string& partition_key : partition_keys) {
+ auto iter = static_partition_.find(partition_key);
+ if (iter == static_partition_.end()) {
+ break;
+ }
+ ordered_partition.emplace_back(partition_key, iter->second);
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::string partition_path,
+ PartitionPathUtils::GeneratePartitionPath(
+ ordered_partition, table_->PartitionOnlyValueInPath()));
+ std::string directory = PathUtil::JoinPath(table_->Location(), partition_path);
+ // The spec may name a partition no message covers: an overwrite of a directory a scan
+ // skips would clear files that are not this table's data.
+ PAIMON_RETURN_NOT_OK(FormatPathValidation::ValidateDirectoryIsVisible(table_, directory,
+ "static partition"));
+ PAIMON_ASSIGN_OR_RAISE(bool exists, file_system->Exists(directory));
+ if (!exists) {
+ // Nothing to clear, but created regardless: an overwrite leaves an empty partition
+ // behind rather than removing it from the table.
+ PAIMON_RETURN_NOT_OK(file_system->Mkdirs(directory));
+ } else if (overwrite_) {
+ PAIMON_RETURN_NOT_OK(DeletePreviousDataFiles(
+ directory, static_cast<int32_t>(partition_keys.size() - ordered_partition.size())));
+ replaced_directories.push_back(std::move(directory));
+ }
+ } else if (overwrite_ && ReplacesOnlyWrittenPartitions()) {
+ // The directory the message's partition names, not the file's own parent: a file may sit
+ // below its partition directory, and an overwrite replaces the whole partition.
+ std::set<std::string> directories;
+ for (const FormatCommitMessage& message : commit_messages) {
+ PAIMON_ASSIGN_OR_RAISE(
+ std::string directory,
+ FormatPathValidation::BuildPartitionDirectory(table_, message.partition));
+ directories.insert(std::move(directory));
+ }
+ for (const std::string& directory : directories) {
+ // A complete partition directory, so no partition level is left below it.
+ PAIMON_RETURN_NOT_OK(DeletePreviousDataFiles(directory, /*partition_levels=*/0));
+ replaced_directories.push_back(directory);
+ }
+ } else if (overwrite_) {
+ // Replacing the table rather than the partitions this commit wrote: unpartitioned, or
+ // `dynamic-partition-overwrite` off. What goes is everything the table holds, so a commit
+ // carrying no message still empties it - Java's `tableDataDirectories()`.
+ PAIMON_RETURN_NOT_OK(DeletePreviousDataFiles(table_->Location(),
+ static_cast<int32_t>(partition_keys.size())));
+ replaced_directories.push_back(table_->Location());
+ }
+ if (overwrite_) {
+ // Nothing else records what an overwrite emptied. Only a sample of the paths, since a
+ // table of many partitions would put every directory in one line.
+ constexpr size_t kLoggedDirectories = 5;
+ const char* replaced_what = !static_partition_.empty() ? "the static partition"
+ : ReplacesOnlyWrittenPartitions()
+ ? "the partitions this commit writes"
+ : "the whole table";
+ const size_t logged = std::min(replaced_directories.size(), kLoggedDirectories);
+ std::vector<std::string> sample(
+ replaced_directories.begin(),
+ replaced_directories.begin() + static_cast<std::ptrdiff_t>(logged));
+ PAIMON_LOG_INFO(
+ CommitLogger(),
+ "Cleared the data files of %zu directories of format table %s, replacing %s: %s%s",
+ replaced_directories.size(), table_->FullName().c_str(), replaced_what,
+ fmt::format("{}", fmt::join(sample, ", ")).c_str(),
+ replaced_directories.size() > logged ? ", ..." : "");
+ }
+
+ // Recorded before its rename, not after: a failed rename may have published the file all the
+ // same - on an object store it is a copy followed by a delete - and a target left behind reads
+ // back as this table's data. Removing it is safe: the name carries the staging write's uuid.
+ std::vector<std::string> attempted_publish_paths;
+ attempted_publish_paths.reserve(commit_messages.size());
+ for (const FormatCommitMessage& message : commit_messages) {
+ attempted_publish_paths.push_back(message.file_path);
+ Status status = file_system->Rename(message.temp_file_path, message.file_path);
+ if (!status.ok()) {
+ // Only when nothing was replaced, so the table holds all of this write or none of it.
+ // An overwrite has already deleted what it replaces, so taking its files back would
+ // leave neither the old rows nor the new - Java's
+ // `markPublishedTargetsToPreserveOnAbort()` preserves them for the same reason.
+ if (!overwrite_) {
+ for (const std::string& file_path : attempted_publish_paths) {
+ Status rollback = file_system->Delete(file_path, /*recursive=*/false);
+ // Never published is the ordinary case for the target the rename failed on.
+ if (!rollback.ok() && !rollback.IsNotExist()) {
+ PAIMON_LOG_WARN(CommitLogger(),
+ "Failed to take back the published file %s: %s",
+ file_path.c_str(), rollback.ToString().c_str());
+ }
+ }
+ }
+ return Status::IOError(fmt::format("failed to commit {} of format table {}: {}",
+ message.ToString(), table_->FullName(),
+ status.ToString()));
+ }
+ }
+ // Publishing is a loop of renames, not one atomic step: without this line a commit that
+ // started and never finished would look from the log like one that did.
+ PAIMON_LOG_INFO(CommitLogger(), "Finished %s %zu files to format table %s, duration %lu ms",
+ overwrite_ ? "overwriting with" : "committing", commit_messages.size(),
+ table_->FullName().c_str(), duration.Get());
+ return Status::OK();
+}
+
+Status FormatTableCommit::Abort(const std::vector<FormatCommitMessage>& commit_messages) {
+ std::shared_ptr<FileSystem> file_system = table_->GetFileSystem();
+ // Best effort: a file that cannot be removed stays behind under its hidden name, where a scan
+ // ignores it. Java's `abort()` reports its failures instead; here the log is the signal, so
+ // one bad message cannot stop the rest being cleaned up.
+ for (const FormatCommitMessage& message : commit_messages) {
+ // Returning at the first bad one would strand the staged files of the good ones.
+ Status valid = ValidateCommitMessage(message, table_, static_partition_);
+ if (!valid.ok()) {
+ PAIMON_LOG_WARN(CommitLogger(),
+ "Refusing to discard a file that does not describe this table: %s",
+ valid.ToString().c_str());
+ continue;
+ }
+ Status status = file_system->Delete(message.temp_file_path, /*recursive=*/false);
+ // Already gone is the expected case after a rollback took the file back.
+ if (!status.ok() && !status.IsNotExist()) {
+ PAIMON_LOG_WARN(CommitLogger(), "Failed to discard the staged file %s: %s",
+ message.temp_file_path.c_str(), status.ToString().c_str());
+ }
+ }
+ return Status::OK();
+}
+
+} // namespace paimon
diff --git a/src/paimon/core/table/format/format_table_commit.h b/src/paimon/core/table/format/format_table_commit.h
new file mode 100644
index 00000000..bfc63d40
--- /dev/null
+++ b/src/paimon/core/table/format/format_table_commit.h
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "paimon/core/table/format/format_commit_message.h"
+#include "paimon/result.h"
+#include "paimon/status.h"
+#include "paimon/table/format/format_table.h"
+
+namespace paimon {
+
+/// Publishes the files a `FormatTableWrite` produced, by renaming each out of the `_temporary`
+/// directory it was staged in.
+///
+/// A directory has no metadata to switch, so a commit is not atomic across files: a reader
+/// scanning midway sees the ones renamed so far, though each file becomes visible whole. A commit
+/// that fails partway takes back every target it tried to publish, best effort; an overwrite
+/// cannot, having deleted what it replaces first.
+///
+/// Only this job's own messages may be passed in: the checks here can tell that a message
+/// describes this table, not whose staged file it names. Not thread-safe, and two overwriting
+/// commits over the same directory race.
+class FormatTableCommit {
+ public:
+ /// @param overwrite Whether the commit replaces what is already there instead of adding to
+ /// it. What it replaces: the static partition when one is given, otherwise the
+ /// partitions this commit writes when `dynamic-partition-overwrite` is on and the table
+ /// is partitioned, and the whole table otherwise. Replacing the table empties it even
+ /// when no file is published.
+ /// @param static_partition Partition the commit writes to, keyed by partition field name. It
+ /// may name only the leading keys, standing for every partition below that prefix.
+ /// Empty leaves the partitions to the written files.
+ static Result<std::unique_ptr<FormatTableCommit>> Create(
+ const std::shared_ptr<FormatTable>& table, bool overwrite,
+ const std::map<std::string, std::string>& static_partition);
+
+ ~FormatTableCommit();
+
+ /// Renames every written file into place, first clearing what it replaces when the commit
+ /// overwrites.
+ Status Commit(const std::vector<FormatCommitMessage>& commit_messages);
+
+ /// Removes the staged files of `commit_messages` instead of publishing them.
+ ///
+ /// It undoes a commit that never happened, not one that did: a file already renamed into
+ /// place is no longer staged and stays. It never fails, so the log is the only signal that a
+ /// cleanup fell short.
+ Status Abort(const std::vector<FormatCommitMessage>& commit_messages);
+
+ private:
+ FormatTableCommit(const std::shared_ptr<FormatTable>& table, bool overwrite,
+ const std::map<std::string, std::string>& static_partition,
+ bool dynamic_partition_overwrite);
+
+ /// Whether an overwrite naming no partition replaces only the partitions this commit wrote.
+ /// An unpartitioned table has no partitions to select, so it is always replaced whole.
+ bool ReplacesOnlyWrittenPartitions() const;
+
+ /// The body of `Commit()`, so that every failure in it is followed by the same cleanup.
+ Status CommitImpl(const std::vector<FormatCommitMessage>& commit_messages);
+
+ /// Deletes the committed data files under `directory`, leaving another writer's staged files
+ /// alone.
+ ///
+ /// @param partition_levels Directory levels below `directory` that still hold partition
+ /// directories, which a static partition naming only the leading keys leaves behind.
+ Status DeletePreviousDataFiles(const std::string& directory, int32_t partition_levels) const;
+
+ std::shared_ptr<FormatTable> table_;
+ bool overwrite_ = false;
+ std::map<std::string, std::string> static_partition_;
+ /// Read once at `Create()`, so an overwrite cannot see a different value than the table was
+ /// opened with.
+ bool dynamic_partition_overwrite_ = true;
+};
+
+} // namespace paimon
diff --git a/src/paimon/core/table/format/format_table_file_store_commit.cpp b/src/paimon/core/table/format/format_table_file_store_commit.cpp
new file mode 100644
index 00000000..3e53c526
--- /dev/null
+++ b/src/paimon/core/table/format/format_table_file_store_commit.cpp
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/core/table/format/format_table_file_store_commit.h"
+
+#include "fmt/format.h"
+#include "paimon/common/metrics/metrics_impl.h"
+#include "paimon/core/table/format/format_commit_message.h"
+#include "paimon/core/table/format/format_table_commit.h"
+#include "paimon/table/format/format_table.h"
+
+namespace paimon {
+
+namespace {
+
+/// The refusal the snapshot half of `FileStoreCommit` returns, so a caller reading one knows to
+/// look at the table type rather than at its own arguments.
+Status UnsupportedFormatTableOperation(const char* what) {
+ return Status::NotImplemented(
+ fmt::format("a format table has no snapshots or manifests, so {}", what));
+}
+
+} // namespace
+
+FormatTableFileStoreCommit::FormatTableFileStoreCommit(const std::shared_ptr<FormatTable>& table)
+ : table_(table) {}
+
+FormatTableFileStoreCommit::~FormatTableFileStoreCommit() = default;
+
+Result<std::unique_ptr<FormatTableFileStoreCommit>> FormatTableFileStoreCommit::Create(
+ const std::shared_ptr<FormatTable>& table) {
+ if (table == nullptr) {
+ return Status::Invalid("format table commit requires a table");
+ }
+ return std::unique_ptr<FormatTableFileStoreCommit>(new FormatTableFileStoreCommit(table));
+}
+
+Result<std::vector<FormatCommitMessage>> FormatTableFileStoreCommit::ToFormatMessages(
+ const std::vector<std::shared_ptr<CommitMessage>>& commit_messages) {
+ std::vector<FormatCommitMessage> messages;
+ messages.reserve(commit_messages.size());
+ for (const std::shared_ptr<CommitMessage>& commit_message : commit_messages) {
+ auto message = std::dynamic_pointer_cast<FormatCommitMessage>(commit_message);
+ if (message == nullptr) {
+ return Status::Invalid(
+ "a format table commit takes the messages a format table write produced; this one "
+ "describes files to record in a manifest");
+ }
+ messages.push_back(*message);
+ }
+ return messages;
+}
+
+Status FormatTableFileStoreCommit::Commit(
+ const std::vector<std::shared_ptr<CommitMessage>>& commit_messages, int64_t commit_identifier,
+ std::optional<int64_t> watermark) {
+ if (commit_identifier != BATCH_WRITE_COMMIT_IDENTIFIER) {
+ return UnsupportedFormatTableOperation("a commit identifier has nowhere to be recorded");
+ }
+ if (watermark) {
+ return UnsupportedFormatTableOperation("a watermark has nowhere to be recorded");
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::vector<FormatCommitMessage> messages,
+ ToFormatMessages(commit_messages));
+ PAIMON_ASSIGN_OR_RAISE(
+ std::unique_ptr<FormatTableCommit> commit,
+ FormatTableCommit::Create(table_, /*overwrite=*/false, /*static_partition=*/{}));
+ return commit->Commit(messages);
+}
+
+Status FormatTableFileStoreCommit::Overwrite(
+ const std::map<std::string, std::string>& partition,
+ const std::vector<std::shared_ptr<CommitMessage>>& commit_messages, int64_t commit_identifier,
+ std::optional<int64_t> watermark) {
+ if (commit_identifier != BATCH_WRITE_COMMIT_IDENTIFIER) {
+ return UnsupportedFormatTableOperation("a commit identifier has nowhere to be recorded");
+ }
+ if (watermark) {
+ return UnsupportedFormatTableOperation("a watermark has nowhere to be recorded");
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::vector<FormatCommitMessage> messages,
+ ToFormatMessages(commit_messages));
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FormatTableCommit> commit,
+ FormatTableCommit::Create(table_, /*overwrite=*/true, partition));
+ return commit->Commit(messages);
+}
+
+Status FormatTableFileStoreCommit::Abort(
+ const std::vector<std::shared_ptr<CommitMessage>>& commit_messages) {
+ PAIMON_ASSIGN_OR_RAISE(std::vector<FormatCommitMessage> messages,
+ ToFormatMessages(commit_messages));
+ PAIMON_ASSIGN_OR_RAISE(
+ std::unique_ptr<FormatTableCommit> commit,
+ FormatTableCommit::Create(table_, /*overwrite=*/false, /*static_partition=*/{}));
+ return commit->Abort(messages);
+}
+
+Result<int64_t> FormatTableFileStoreCommit::CommitWithProgress(
+ const std::vector<RealtimeCommitProgress>& realtime_commits, int64_t commit_identifier,
+ std::optional<int64_t> watermark) {
+ return UnsupportedFormatTableOperation("it has no real-time offsets to publish with a commit");
+}
+
+Result<int32_t> FormatTableFileStoreCommit::FilterAndCommit(
+ const std::map<int64_t, std::vector<std::shared_ptr<CommitMessage>>>&
+ commit_identifier_and_messages,
+ std::optional<int64_t> watermark) {
+ return UnsupportedFormatTableOperation(
+ "nothing records which commit identifiers have already been committed");
+}
+
+Result<int32_t> FormatTableFileStoreCommit::FilterAndOverwrite(
+ const std::map<std::string, std::string>& partition,
+ const std::vector<std::shared_ptr<CommitMessage>>& commit_messages, int64_t commit_identifier,
+ std::optional<int64_t> watermark) {
+ return UnsupportedFormatTableOperation(
+ "nothing records which commit identifiers have already been committed");
+}
+
+Result<std::string> FormatTableFileStoreCommit::GetLastCommitTableRequest() {
+ return UnsupportedFormatTableOperation(
+ "it commits by renaming files rather than through a rest catalog");
+}
+
+Result<int32_t> FormatTableFileStoreCommit::Expire() {
+ return UnsupportedFormatTableOperation("there are no snapshots to expire");
+}
+
+Status FormatTableFileStoreCommit::DropPartition(
+ const std::vector<std::map<std::string, std::string>>& partitions, int64_t commit_identifier) {
+ // Not refused for lack of snapshots but simply missing: dropping a partition means removing
+ // its directory, which nothing here does yet. An overwrite of that partition with no messages
+ // empties it, which is as far as this goes today.
+ return Status::NotImplemented(
+ "dropping a partition of a format table is not implemented yet; an overwrite of it with "
+ "no commit messages empties it instead");
+}
+
+Status FormatTableFileStoreCommit::TruncateTable(int64_t commit_identifier) {
+ return Status::NotImplemented("emptying a format table is not implemented yet");
+}
+
+Result<bool> FormatTableFileStoreCommit::RollbackToAsLatest(int64_t target_snapshot_id) {
+ return UnsupportedFormatTableOperation("there is no snapshot to roll back to");
+}
+
+FileStoreCommit& FormatTableFileStoreCommit::RowIdCheckConflict(
+ std::optional<int64_t> row_id_check_from_snapshot) {
+ // The interface returns a reference and cannot report a refusal, so this is the one call that
+ // has to be a no-op. A format table records no row ids, so there is no conflict to check.
+ return *this;
+}
+
+std::shared_ptr<Metrics> FormatTableFileStoreCommit::GetCommitMetrics() const {
+ // Empty rather than null, as on the write side's `GetMetrics()`.
+ return std::make_shared<MetricsImpl>();
+}
+
+} // namespace paimon
diff --git a/src/paimon/core/table/format/format_table_file_store_commit.h b/src/paimon/core/table/format/format_table_file_store_commit.h
new file mode 100644
index 00000000..539e45e5
--- /dev/null
+++ b/src/paimon/core/table/format/format_table_file_store_commit.h
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "paimon/core/table/format/format_commit_message.h"
+#include "paimon/defs.h"
+#include "paimon/file_store_commit.h"
+#include "paimon/result.h"
+#include "paimon/status.h"
+
+namespace paimon {
+
+class FormatTable;
+class Metrics;
+
+/// Commits a format table through the `FileStoreCommit` interface, so that a caller holding a
+/// table path commits it the way it commits any other table. Java Paimon does the same through
+/// `FormatTable.newBatchWriteBuilder()`.
+///
+/// Most of `FileStoreCommit` is about snapshots and manifests, which a format table keeps none
+/// of, so each such call is refused rather than quietly doing nothing. `RowIdCheckConflict()` is
+/// the exception, returning a reference with no way to report one.
+class FormatTableFileStoreCommit : public FileStoreCommit {
+ public:
+ static Result<std::unique_ptr<FormatTableFileStoreCommit>> Create(
+ const std::shared_ptr<FormatTable>& table);
+
+ ~FormatTableFileStoreCommit() override;
+
+ Status Commit(const std::vector<std::shared_ptr<CommitMessage>>& commit_messages,
+ int64_t commit_identifier, std::optional<int64_t> watermark) override;
+
+ Status Overwrite(const std::map<std::string, std::string>& partition,
+ const std::vector<std::shared_ptr<CommitMessage>>& commit_messages,
+ int64_t commit_identifier, std::optional<int64_t> watermark) override;
+
+ Status Abort(const std::vector<std::shared_ptr<CommitMessage>>& commit_messages) override;
+
+ Result<int64_t> CommitWithProgress(const std::vector<RealtimeCommitProgress>& realtime_commits,
+ int64_t commit_identifier,
+ std::optional<int64_t> watermark) override;
+
+ Result<int32_t> FilterAndCommit(
+ const std::map<int64_t, std::vector<std::shared_ptr<CommitMessage>>>&
+ commit_identifier_and_messages,
+ std::optional<int64_t> watermark) override;
+
+ Result<int32_t> FilterAndOverwrite(
+ const std::map<std::string, std::string>& partition,
+ const std::vector<std::shared_ptr<CommitMessage>>& commit_messages,
+ int64_t commit_identifier, std::optional<int64_t> watermark) override;
+
+ Result<std::string> GetLastCommitTableRequest() override;
+
+ Result<int32_t> Expire() override;
+
+ Status DropPartition(const std::vector<std::map<std::string, std::string>>& partitions,
+ int64_t commit_identifier) override;
+
+ Status TruncateTable(int64_t commit_identifier) override;
+
+ Result<bool> RollbackToAsLatest(int64_t target_snapshot_id) override;
+
+ FileStoreCommit& RowIdCheckConflict(std::optional<int64_t> row_id_check_from_snapshot) override;
+
+ std::shared_ptr<Metrics> GetCommitMetrics() const override;
+
+ private:
+ explicit FormatTableFileStoreCommit(const std::shared_ptr<FormatTable>& table);
+
+ /// The messages a format table commit takes, or a refusal when one of them belongs to another
+ /// table type. Nothing is published until every message has been recognised.
+ static Result<std::vector<FormatCommitMessage>> ToFormatMessages(
+ const std::vector<std::shared_ptr<CommitMessage>>& commit_messages);
+
+ std::shared_ptr<FormatTable> table_;
+};
+
+} // namespace paimon
diff --git a/src/paimon/core/table/format/format_table_file_store_write.cpp b/src/paimon/core/table/format/format_table_file_store_write.cpp
new file mode 100644
index 00000000..50d417db
--- /dev/null
+++ b/src/paimon/core/table/format/format_table_file_store_write.cpp
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/core/table/format/format_table_file_store_write.h"
+
+#include <utility>
+
+#include "paimon/common/metrics/metrics_impl.h"
+#include "paimon/core/table/format/format_commit_message.h"
+#include "paimon/core/table/format/format_table_write.h"
+#include "paimon/table/format/format_table.h"
+
+namespace paimon {
+
+FormatTableFileStoreWrite::FormatTableFileStoreWrite(std::unique_ptr<FormatTableWrite>&& write)
+ : write_(std::move(write)) {}
+
+FormatTableFileStoreWrite::~FormatTableFileStoreWrite() = default;
+
+Result<std::unique_ptr<FormatTableFileStoreWrite>> FormatTableFileStoreWrite::Create(
+ const std::shared_ptr<FormatTable>& table, const std::shared_ptr<MemoryPool>& pool) {
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FormatTableWrite> write,
+ FormatTableWrite::Create(table, pool));
+ return std::unique_ptr<FormatTableFileStoreWrite>(
+ new FormatTableFileStoreWrite(std::move(write)));
+}
+
+Status FormatTableFileStoreWrite::Write(std::unique_ptr<RecordBatch>&& batch) {
+ if (write_ == nullptr) {
+ return Status::Invalid("format table write has been closed");
+ }
+ return write_->Write(std::move(batch));
+}
+
+Status FormatTableFileStoreWrite::Compact(const std::map<std::string, std::string>& partition,
+ int32_t bucket, bool full_compaction) {
+ return Status::NotImplemented(
+ "a format table cannot be compacted: it has no manifests to rewrite and no buckets to "
+ "compact within");
+}
+
+Result<std::vector<std::shared_ptr<CommitMessage>>> FormatTableFileStoreWrite::PrepareCommit(
+ bool wait_compaction, int64_t commit_identifier) {
+ if (write_ == nullptr) {
+ return Status::Invalid("format table write has been closed");
+ }
+ // `wait_compaction` asks to wait rather than to do anything, and there is no compaction here
+ // to wait for, so it is honoured by returning at once rather than refused.
+ //
+ // A commit identifier is how a streaming write tells its attempts apart in a snapshot; a
+ // format table has no snapshots to record one in, so only a batch write fits here.
+ if (commit_identifier != BATCH_WRITE_COMMIT_IDENTIFIER) {
+ return Status::NotImplemented(
+ "a format table takes batch writes only: it has no snapshot to record a commit "
+ "identifier in");
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::vector<FormatCommitMessage> messages, write_->PrepareCommit());
+ std::vector<std::shared_ptr<CommitMessage>> result;
+ result.reserve(messages.size());
+ for (const FormatCommitMessage& message : messages) {
+ result.push_back(std::make_shared<FormatCommitMessage>(message));
+ }
+ return result;
+}
+
+std::shared_ptr<Metrics> FormatTableFileStoreWrite::GetMetrics() const {
+ // Empty rather than null, so a caller merging metrics from several writers need not tell a
+ // table type that keeps none apart from one that does.
+ return std::make_shared<MetricsImpl>();
+}
+
+Status FormatTableFileStoreWrite::Close() {
+ // Closing drops the write rather than aborting it: a write that never prepared a commit clears
+ // what it staged from its own destructor, while one that has prepared has handed those files
+ // to a commit and must leave them alone.
+ write_.reset();
+ return Status::OK();
+}
+
+} // namespace paimon
diff --git a/src/paimon/core/table/format/format_table_file_store_write.h b/src/paimon/core/table/format/format_table_file_store_write.h
new file mode 100644
index 00000000..2e6f49c0
--- /dev/null
+++ b/src/paimon/core/table/format/format_table_file_store_write.h
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "paimon/defs.h"
+#include "paimon/file_store_write.h"
+#include "paimon/result.h"
+#include "paimon/status.h"
+
+namespace paimon {
+
+class FormatTable;
+class FormatTableWrite;
+class MemoryPool;
+class Metrics;
+
+/// Writes a format table through the `FileStoreWrite` interface, so that a caller holding a table
+/// path writes it the way it writes any other table. Java Paimon does the same through
+/// `FormatTable.newBatchWriteBuilder()`.
+///
+/// The parts of `FileStoreWrite` about manifests and snapshots - compaction, and the commit
+/// identifier a streaming write carries - have nothing here to act on and are refused.
+class FormatTableFileStoreWrite : public FileStoreWrite {
+ public:
+ static Result<std::unique_ptr<FormatTableFileStoreWrite>> Create(
+ const std::shared_ptr<FormatTable>& table, const std::shared_ptr<MemoryPool>& pool);
+
+ ~FormatTableFileStoreWrite() override;
+
+ Status Write(std::unique_ptr<RecordBatch>&& batch) override;
+
+ Status Compact(const std::map<std::string, std::string>& partition, int32_t bucket,
+ bool full_compaction) override;
+
+ Result<std::vector<std::shared_ptr<CommitMessage>>> PrepareCommit(
+ bool wait_compaction, int64_t commit_identifier) override;
+
+ std::shared_ptr<Metrics> GetMetrics() const override;
+
+ Status Close() override;
+
+ private:
+ explicit FormatTableFileStoreWrite(std::unique_ptr<FormatTableWrite>&& write);
+
+ std::unique_ptr<FormatTableWrite> write_;
+};
+
+} // namespace paimon
diff --git a/src/paimon/core/table/format/format_table_loader.cpp b/src/paimon/core/table/format/format_table_loader.cpp
new file mode 100644
index 00000000..c50b85e4
--- /dev/null
+++ b/src/paimon/core/table/format/format_table_loader.cpp
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/core/table/format/format_table_loader.h"
+
+#include <cassert>
+#include <optional>
+
+#include "fmt/format.h"
+#include "paimon/catalog/identifier.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/common/utils/path_util.h"
+#include "paimon/core/options/table_type.h"
+#include "paimon/core/schema/schema_manager.h"
+#include "paimon/core/schema/table_schema.h"
+#include "paimon/core/utils/branch_manager.h"
+#include "paimon/schema/schema.h"
+#include "paimon/table/format/format_table.h"
+
+namespace paimon {
+
+Result<std::shared_ptr<FormatTable>> FormatTableLoader::TryLoad(
+ const std::shared_ptr<FileSystem>& file_system, const std::string& table_path,
+ const std::string& branch, const std::map<std::string, std::string>& options,
+ const std::optional<std::string>& specific_table_schema, const SchemaManager* schema_manager,
+ std::shared_ptr<TableSchema>* table_schema_out) {
+ assert(table_schema_out != nullptr);
+ table_schema_out->reset();
+
+ std::shared_ptr<TableSchema> table_schema;
+ if (branch == BranchManager::DEFAULT_MAIN_BRANCH && specific_table_schema) {
+ // Handing the schema over only saves reading it. A context always names a table path and
+ // this library writes the metadata under it, so `schema` and `branch` stay metadata.
+ PAIMON_ASSIGN_OR_RAISE(table_schema,
+ TableSchema::CreateFromJson(specific_table_schema.value()));
+ } else {
+ // Through the caller's manager when it has one, so that the read warms the cache it goes
+ // on to use rather than a cache that dies with this call.
+ SchemaManager own_schema_manager(file_system, table_path, branch);
+ const SchemaManager& reader =
+ schema_manager != nullptr ? *schema_manager : own_schema_manager;
+ PAIMON_ASSIGN_OR_RAISE(std::optional<std::shared_ptr<TableSchema>> latest_schema,
+ reader.Latest());
+ if (!latest_schema) {
+ return std::shared_ptr<FormatTable>();
+ }
+ table_schema = *latest_schema;
+ }
+ *table_schema_out = table_schema;
+
+ // From the schema alone: see `FormatTable::Create()`.
+ PAIMON_ASSIGN_OR_RAISE(bool is_format_table,
+ TableTypeDefine::IsFormatTable(table_schema->Options()));
+ if (!is_format_table) {
+ return std::shared_ptr<FormatTable>();
+ }
+ // A format table has no branches, so it is refused rather than served from the table path
+ // under another branch's name. See `SchemaValidation::ValidateFormatTableSchema()`.
+ if (BranchManager::NormalizeBranch(branch) != BranchManager::DEFAULT_MAIN_BRANCH) {
+ return Status::Invalid(
+ fmt::format("a format table has no branches, so {} cannot be read on branch '{}'",
+ table_path, branch));
+ }
+ // A context names a path, never a catalog identifier, so the table is known by the directory
+ // it sits in.
+ return FormatTable::Create(file_system, table_path, Identifier(PathUtil::GetName(table_path)),
+ checked_pointer_cast<DataSchema>(table_schema),
+ /*location_carries_paimon_metadata=*/true, options);
+}
+
+} // namespace paimon
diff --git a/src/paimon/core/table/format/format_table_loader.h b/src/paimon/core/table/format/format_table_loader.h
new file mode 100644
index 00000000..290ee94a
--- /dev/null
+++ b/src/paimon/core/table/format/format_table_loader.h
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <map>
+#include <memory>
+#include <optional>
+#include <string>
+
+#include "paimon/result.h"
+
+namespace paimon {
+
+class FileSystem;
+class FormatTable;
+class SchemaManager;
+class TableSchema;
+
+/// Recognises a format table behind a table path.
+///
+/// The scan, read, write and commit entry points all take a path and have to answer the same
+/// question before they can dispatch, so one place answers it and they cannot disagree.
+class FormatTableLoader {
+ public:
+ FormatTableLoader() = delete;
+ ~FormatTableLoader() = delete;
+
+ /// Loads `table_path` as a format table, or returns null when it is not one.
+ ///
+ /// Null is also the answer when there is no schema under the path at all: what to say about a
+ /// table that is not there is the managed path's to decide, and it says it in more detail.
+ ///
+ /// The table type is read from the schema, never from `options`: `type` is structural, and one
+ /// call's options must not decide what kind of table this is.
+ ///
+ /// @param branch Branch the schema is read from. Take it from the same place the managed path
+ /// in the same entry point takes it, or the two dispatch on different schemas. Anything
+ /// but the main branch is refused once the schema turns out to be a format table's,
+ /// which has no branches to read.
+ /// @param options Options given at the call, which win over the ones the schema stored.
+ /// @param specific_table_schema A schema the caller already holds, as json, which saves
+ /// reading one from under the path. Used only on the main branch, as on the managed
+ /// read path.
+ /// @param schema_manager The manager to read the schema through, or null to use one of this
+ /// call's own. A caller that has one must pass it: a manager caches what it read.
+ /// @param table_schema_out Set to the schema this read, or null when there is none, so that a
+ /// caller falling back to the managed path does not read it twice.
+ static Result<std::shared_ptr<FormatTable>> TryLoad(
+ const std::shared_ptr<FileSystem>& file_system, const std::string& table_path,
+ const std::string& branch, const std::map<std::string, std::string>& options,
+ const std::optional<std::string>& specific_table_schema,
+ const SchemaManager* schema_manager, std::shared_ptr<TableSchema>* table_schema_out);
+};
+
+} // namespace paimon
diff --git a/src/paimon/core/table/format/format_table_read.cpp b/src/paimon/core/table/format/format_table_read.cpp
new file mode 100644
index 00000000..7bc14b59
--- /dev/null
+++ b/src/paimon/core/table/format/format_table_read.cpp
@@ -0,0 +1,617 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/core/table/format/format_table_read.h"
+
+#include <algorithm>
+#include <map>
+#include <set>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "arrow/api.h"
+#include "arrow/c/bridge.h"
+#include "arrow/c/helpers.h"
+#include "fmt/format.h"
+#include "paimon/common/data/binary_row.h"
+#include "paimon/common/predicate/predicate_validator.h"
+#include "paimon/common/reader/complete_row_kind_batch_reader.h"
+#include "paimon/common/reader/concat_batch_reader.h"
+#include "paimon/common/reader/data_file_reader_factory.h"
+#include "paimon/common/reader/predicate_batch_reader.h"
+#include "paimon/common/types/data_field.h"
+#include "paimon/common/utils/arrow/mem_utils.h"
+#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/binary_row_partition_computer.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/common/utils/scope_guard.h"
+#include "paimon/core/core_options.h"
+#include "paimon/core/io/field_mapping_reader.h"
+#include "paimon/core/table/format/format_data_split.h"
+#include "paimon/core/table/format/format_path_validation.h"
+#include "paimon/core/table/format/lazy_concat_batch_reader.h"
+#include "paimon/core/utils/field_mapping.h"
+#include "paimon/format/file_format.h"
+#include "paimon/format/file_format_factory.h"
+#include "paimon/fs/file_system.h"
+#include "paimon/predicate/predicate.h"
+
+namespace paimon {
+
+namespace {
+
+/// Whether a `TIMESTAMP` a file of `format` stores in `file_unit` may be read as the table's
+/// `table_unit`. A file's unit means something else in each format, so the rule does too:
+///
+/// - parquet records the unit, so the file's is the one its values are in. Only `MILLI` under a
+/// `SECOND` column is rescaled, by `ParquetTimestampConverter`, which parquet's own round trip
+/// needs since it has no `SECOND`. Any other pair would be relabelled rather than rescaled -
+/// `us` under an `ms` type reads a thousand times too large - so it is refused;
+/// - orc records none: a value is a seconds-plus-nanoseconds pair that `OrcAdapter` builds at
+/// whatever precision the read schema asks for, and its type converter reports `NANO` either
+/// way. Comparing that would refuse every orc `TIMESTAMP(0)`, `(3)` and `(6)`, this library's
+/// own writes among them.
+bool TimestampUnitIsCompatible(arrow::TimeUnit::type file_unit, arrow::TimeUnit::type table_unit,
+ FormatTable::Format format) {
+ if (file_unit == table_unit) {
+ return true;
+ }
+ switch (format) {
+ case FormatTable::Format::ORC:
+ return true;
+ case FormatTable::Format::PARQUET:
+ return file_unit == arrow::TimeUnit::MILLI && table_unit == arrow::TimeUnit::SECOND;
+ }
+ return false;
+}
+
+/// Whether a file storing `file_type` may be read as the table's `table_type`, for a file of
+/// `format`.
+///
+/// The two have to describe the same values, with three exceptions, each allowed because something
+/// downstream already reconciles it:
+///
+/// - a `TIMESTAMP` may differ in timezone, which the read relabels without touching the instant
+/// the value names, and in unit, for which see `TimestampUnitIsCompatible()`;
+/// - a `STRUCT` in the file may hold children the table does not name, which are read past as an
+/// unnamed top-level column is. The ones it does name must come in the table's own order: a
+/// nested column is read where the file stores it and nothing downstream puts it back, so a
+/// struct ordered differently would return its values under the wrong names;
+/// - a `LIST` or `MAP` element may be named differently, since what an engine calls one varies.
+bool FileTypeMatchesTableType(const std::shared_ptr<arrow::DataType>& file_type,
+ const std::shared_ptr<arrow::DataType>& table_type,
+ FormatTable::Format format) {
+ if (file_type->id() != table_type->id()) {
+ return false;
+ }
+ switch (file_type->id()) {
+ case arrow::Type::STRUCT: {
+ int32_t file_index = 0;
+ for (const std::shared_ptr<arrow::Field>& table_child : table_type->fields()) {
+ bool found = false;
+ while (file_index < file_type->num_fields()) {
+ const std::shared_ptr<arrow::Field>& file_child =
+ file_type->field(file_index++);
+ if (file_child->name() != table_child->name()) {
+ continue;
+ }
+ if (!FileTypeMatchesTableType(file_child->type(), table_child->type(),
+ format)) {
+ return false;
+ }
+ found = true;
+ break;
+ }
+ if (!found) {
+ return false;
+ }
+ }
+ return true;
+ }
+ case arrow::Type::LIST:
+ case arrow::Type::MAP: {
+ if (file_type->num_fields() != table_type->num_fields()) {
+ return false;
+ }
+ for (int32_t i = 0; i < file_type->num_fields(); i++) {
+ if (!FileTypeMatchesTableType(file_type->field(i)->type(),
+ table_type->field(i)->type(), format)) {
+ return false;
+ }
+ }
+ return true;
+ }
+ case arrow::Type::TIMESTAMP:
+ return TimestampUnitIsCompatible(
+ checked_pointer_cast<arrow::TimestampType>(file_type)->unit(),
+ checked_pointer_cast<arrow::TimestampType>(table_type)->unit(), format);
+ default:
+ return file_type->Equals(table_type);
+ }
+}
+
+/// One file's columns, described as the table's own: matched by name, in the order the file holds
+/// them, plus the partition columns. A column the table does not name is left out, and a column
+/// the table names that this file does not hold is absent, which is how the field mapping learns
+/// to read that one as null.
+///
+/// The file therefore decides which columns there are and in what order, and the table decides
+/// what each of them is. That is what lets a file written before a column was added still be read,
+/// and one whose columns sit in another order be read in the order it stores them.
+///
+/// Only the columns `read_file_columns` names are held to the table's schema: a partition column
+/// is rebuilt from the directory whether or not the file stores one, and a column outside the
+/// projection is never read, so neither has to agree with the table for this read to be sound.
+///
+/// Of a column that is read, two things are checked here rather than left to the format reader.
+/// Its type, because the reader catches only a difference in kind - parquet reads a column at its
+/// file type and only ever casts timestamps, so a `DECIMAL` of another scale would come back as
+/// the file stored it. And its absence, when the table declares it `NOT NULL`, since the mapping
+/// fills a missing column with nulls.
+Result<std::vector<DataField>> DescribeFileColumns(
+ const std::shared_ptr<arrow::Schema>& file_schema, const std::vector<DataField>& table_fields,
+ const std::vector<std::string>& partition_keys, const std::set<std::string>& read_file_columns,
+ FormatTable::Format format, const std::string& file_path) {
+ std::map<std::string, const DataField*> table_fields_by_name;
+ for (const DataField& table_field : table_fields) {
+ table_fields_by_name.emplace(table_field.Name(), &table_field);
+ }
+
+ std::vector<DataField> file_fields;
+ file_fields.reserve(table_fields.size());
+ std::set<std::string> matched;
+ for (const std::shared_ptr<arrow::Field>& file_field : file_schema->fields()) {
+ auto iter = table_fields_by_name.find(file_field->name());
+ if (iter == table_fields_by_name.end()) {
+ // A column of the file the table does not name: another engine's, and not this
+ // table's data. It is left in the file rather than read past.
+ continue;
+ }
+ if (!matched.insert(file_field->name()).second) {
+ return Status::Invalid(
+ fmt::format("{} holds the column '{}' twice, so which of the two the table's "
+ "column of that name would be read from is undecided",
+ file_path, file_field->name()));
+ }
+ if (read_file_columns.count(file_field->name()) > 0 &&
+ !FileTypeMatchesTableType(file_field->type(), iter->second->Type(), format)) {
+ return Status::Invalid(fmt::format(
+ "{} stores the column '{}' as {}, but the table declares it as {}: a format table "
+ "reads a column as it declares it, having no schema history to convert through",
+ file_path, file_field->name(), file_field->type()->ToString(),
+ iter->second->Type()->ToString()));
+ }
+ file_fields.push_back(*iter->second);
+ }
+ for (const DataField& table_field : table_fields) {
+ if (read_file_columns.count(table_field.Name()) == 0 ||
+ matched.count(table_field.Name()) > 0) {
+ continue;
+ }
+ if (!table_field.ArrowField()->nullable()) {
+ return Status::Invalid(fmt::format(
+ "{} does not hold the column '{}', which the table declares NOT NULL: a column a "
+ "file leaves out is read as null, which that column cannot be",
+ file_path, table_field.Name()));
+ }
+ }
+ // A partition column is rebuilt from the directory name rather than read, so it belongs to
+ // every file whether or not that file stores one of its own. Left out here, it would be read
+ // back as null instead of as the value its directory spells out.
+ for (const DataField& table_field : table_fields) {
+ const bool is_partition_key = std::find(partition_keys.begin(), partition_keys.end(),
+ table_field.Name()) != partition_keys.end();
+ if (is_partition_key && matched.count(table_field.Name()) == 0) {
+ file_fields.push_back(table_field);
+ }
+ }
+ return file_fields;
+}
+
+} // namespace
+
+/// Readers, outermost first: CompleteRowKindBatchReader -> (PredicateBatchReader)
+/// -> LazyConcatBatchReader across the split's files -> FieldMappingReader
+/// -> (DelegatingPrefetchReader) -> (PrefetchFileBatchReader) -> FormatReader
+///
+/// The same shape the managed table path builds, minus what a format table has none of: no
+/// deletion vectors, no bitmap index, no row-tracking fields and no shredding. The last three
+/// readers are built by `DataFileReaderFactory`, which is where the two paths meet.
+class FormatTableRead::Impl {
+ public:
+ std::shared_ptr<FormatTable> table;
+ /// Columns the reader returns, in the order it returns them.
+ std::shared_ptr<arrow::Schema> read_schema;
+ /// The whole table schema, as the mapping describes a column: what a file's own columns are
+ /// matched against, one file at a time. See `DescribeFileColumns()`.
+ ///
+ /// Shared rather than held by value: the callback that opens each file of a split captures
+ /// it, and a copy per file would grow with files times columns. Shared rather than referenced,
+ /// so that a reader still outlives the `FormatTableRead` that built it.
+ std::shared_ptr<const std::vector<DataField>> table_fields;
+ /// The columns this read asks a file for: what it projects, less the partition keys it
+ /// rebuilds from the directory. Only these are held to the table's schema. Shared as
+ /// `table_fields` is, and for the same reason.
+ std::shared_ptr<const std::set<std::string>> read_file_columns;
+ /// Splits the read schema into file columns and partition columns and rewrites the predicate
+ /// against the file's own fields. The same builder the managed table path uses.
+ std::shared_ptr<FieldMappingBuilder> field_mapping_builder;
+ /// Turns a split's partition values into the `BinaryRow` a `FieldMappingReader` fills its
+ /// partition columns from. Null when the table is not partitioned.
+ std::shared_ptr<BinaryRowPartitionComputer> partition_computer;
+ /// The predicate the returned reader applies exactly, or null when the caller filters itself.
+ std::shared_ptr<Predicate> filter_predicate;
+ std::shared_ptr<MemoryPool> pool;
+ /// Derived from `pool` once, so a batch that outlives the reader that made it keeps it alive.
+ std::shared_ptr<arrow::MemoryPool> arrow_pool;
+ /// Runs the reads a prefetching reader issues ahead. Null when nothing asked for prefetch.
+ std::shared_ptr<Executor> executor;
+ std::string format_identifier;
+ /// What a file is opened with: the same struct the managed table path fills in.
+ DataFileReadOptions read_options;
+};
+
+FormatTableRead::FormatTableRead(std::unique_ptr<Impl> impl) : impl_(std::move(impl)) {}
+
+FormatTableRead::~FormatTableRead() = default;
+
+Result<std::unique_ptr<FormatTableRead>> FormatTableRead::TEST_Create(
+ const std::shared_ptr<FormatTable>& table,
+ const std::optional<std::vector<std::string>>& projection,
+ const std::shared_ptr<MemoryPool>& pool, const std::shared_ptr<Predicate>& predicate,
+ bool enable_predicate_filter) {
+ // No context, so nothing asked for prefetch or a cache: a file is opened plainly.
+ return CreateInternal(table, projection, pool, predicate, enable_predicate_filter,
+ DataFileReadOptions(), /*executor=*/nullptr);
+}
+
+Result<std::unique_ptr<FormatTableRead>> FormatTableRead::Create(
+ const std::shared_ptr<FormatTable>& table, const std::shared_ptr<ReadContext>& read_context) {
+ if (table == nullptr) {
+ return Status::Invalid("format table read requires a table");
+ }
+ if (read_context == nullptr) {
+ return Status::Invalid("format table read requires a read context");
+ }
+ if (read_context->GetRealtimeContext() != nullptr) {
+ return Status::NotImplemented(
+ "a format table has no real-time store to union with what is on disk");
+ }
+ // A projected read schema can rename a column, prune a nested one and give it metadata of its
+ // own, while a format table's projection is a list of top-level names.
+ if (read_context->GetReadSchema() != nullptr) {
+ return Status::NotImplemented(
+ "a format table read does not take a projected read schema; name the columns to read "
+ "instead");
+ }
+
+ std::optional<std::vector<std::string>> projection;
+ if (!read_context->GetReadFieldNames().empty()) {
+ projection = read_context->GetReadFieldNames();
+ } else if (!read_context->GetReadFieldIds().empty()) {
+ // Resolved against the table's own schema, which is the only thing that knows the ids: a
+ // file another engine wrote carries none.
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> c_schema, table->GetArrowSchema());
+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> table_schema,
+ arrow::ImportSchema(c_schema.get()));
+ PAIMON_ASSIGN_OR_RAISE(std::vector<DataField> fields,
+ DataField::ConvertArrowSchemaToDataFields(table_schema));
+ std::map<int32_t, std::string> name_by_id;
+ for (const DataField& field : fields) {
+ name_by_id.emplace(field.Id(), field.Name());
+ }
+ std::vector<std::string> names;
+ names.reserve(read_context->GetReadFieldIds().size());
+ for (int32_t field_id : read_context->GetReadFieldIds()) {
+ auto iter = name_by_id.find(field_id);
+ if (iter == name_by_id.end()) {
+ return Status::Invalid(fmt::format("field id {} is not a column of table {}",
+ field_id, table->FullName()));
+ }
+ names.push_back(iter->second);
+ }
+ projection = std::move(names);
+ }
+
+ // Read out of the context here, so `CreateInternal()` has one source for each setting.
+ DataFileReadOptions read_options;
+ read_options.cache = read_context->GetCache();
+ read_options.prefetch_enabled = read_context->EnablePrefetch();
+ read_options.prefetch_max_parallel_num = read_context->GetPrefetchMaxParallelNum();
+ read_options.prefetch_batch_count = read_context->GetPrefetchBatchCount();
+ read_options.read_ahead_cache_enabled = read_context->ReadAheadCacheEnabled();
+ read_options.cache_config = read_context->GetCacheConfig();
+
+ return CreateInternal(table, projection, read_context->GetMemoryPool(),
+ read_context->GetPredicate(), read_context->EnablePredicateFilter(),
+ read_options, read_context->GetExecutor());
+}
+
+Result<std::unique_ptr<FormatTableRead>> FormatTableRead::CreateInternal(
+ const std::shared_ptr<FormatTable>& table,
+ const std::optional<std::vector<std::string>>& projection,
+ const std::shared_ptr<MemoryPool>& pool, const std::shared_ptr<Predicate>& predicate,
+ bool enable_predicate_filter, const DataFileReadOptions& read_options,
+ const std::shared_ptr<Executor>& executor) {
+ if (table == nullptr) {
+ return Status::Invalid("format table read requires a table");
+ }
+ std::shared_ptr<MemoryPool> memory_pool = pool != nullptr ? pool : GetDefaultPool();
+
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> c_schema, table->GetArrowSchema());
+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> table_schema,
+ arrow::ImportSchema(c_schema.get()));
+
+ const std::vector<std::string>& partition_keys = table->PartitionKeys();
+ auto is_partition_key = [&partition_keys](const std::string& name) {
+ return std::find(partition_keys.begin(), partition_keys.end(), name) !=
+ partition_keys.end();
+ };
+
+ arrow::FieldVector read_fields;
+ if (projection) {
+ read_fields.reserve(projection->size());
+ std::set<std::string> projected;
+ for (const std::string& name : *projection) {
+ // A read column is looked up by name, so twice has no meaning to act on.
+ if (!projected.insert(name).second) {
+ return Status::Invalid(fmt::format(
+ "column '{}' appears more than once in the projection, which paimon-cpp does "
+ "not allow",
+ name));
+ }
+ std::shared_ptr<arrow::Field> field = table_schema->GetFieldByName(name);
+ if (field == nullptr) {
+ return Status::Invalid(
+ fmt::format("field '{}' is not a column of table {}", name, table->FullName()));
+ }
+ read_fields.push_back(std::move(field));
+ }
+ } else {
+ read_fields = table_schema->fields();
+ }
+ if (read_fields.empty()) {
+ return Status::Invalid("format table read requires at least one column to read");
+ }
+
+ auto impl = std::make_unique<Impl>();
+ impl->table = table;
+ impl->read_schema = arrow::schema(read_fields);
+ impl->pool = memory_pool;
+ impl->arrow_pool = GetArrowPool(memory_pool);
+ impl->format_identifier = FormatTable::FormatToString(table->GetFormat());
+ PAIMON_ASSIGN_OR_RAISE(std::vector<DataField> table_fields,
+ DataField::ConvertArrowSchemaToDataFields(table_schema));
+ impl->table_fields = std::make_shared<const std::vector<DataField>>(std::move(table_fields));
+ std::set<std::string> read_file_columns;
+ for (const std::shared_ptr<arrow::Field>& read_field : read_fields) {
+ if (!is_partition_key(read_field->name())) {
+ read_file_columns.insert(read_field->name());
+ }
+ }
+ impl->read_file_columns =
+ std::make_shared<const std::set<std::string>>(std::move(read_file_columns));
+
+ const bool has_non_partition_column =
+ std::any_of(table_schema->fields().begin(), table_schema->fields().end(),
+ [&is_partition_key](const std::shared_ptr<arrow::Field>& field) {
+ return !is_partition_key(field->name());
+ });
+ if (!has_non_partition_column) {
+ return Status::Invalid(
+ fmt::format("format table {} has no non-partition column, so its files hold nothing to "
+ "read",
+ table->FullName()));
+ }
+
+ if (predicate != nullptr) {
+ // The same rules `InternalReadContext` applies to a managed table's predicate. The field
+ // index is not among them: everything downstream resolves a field by name.
+ PAIMON_RETURN_NOT_OK(PredicateValidator::ValidatePredicateWithSchema(
+ *impl->read_schema, predicate, /*validate_field_idx=*/false));
+ PAIMON_RETURN_NOT_OK(PredicateValidator::ValidatePredicateWithLiterals(predicate));
+ if (enable_predicate_filter) {
+ impl->filter_predicate = predicate;
+ }
+ }
+
+ // The builder also hands the file reader only the conjuncts naming columns the file holds.
+ PAIMON_ASSIGN_OR_RAISE(
+ std::unique_ptr<FieldMappingBuilder> field_mapping_builder,
+ FieldMappingBuilder::Create(impl->read_schema, partition_keys, predicate));
+ impl->field_mapping_builder = std::move(field_mapping_builder);
+
+ PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options,
+ CoreOptions::FromMap(table->Options(), table->GetFileSystem()));
+ // What the caller decided, with the table's own options filled in over the top: those three
+ // fields describe the table rather than the read, so a caller cannot disagree with them.
+ impl->read_options = read_options;
+ impl->read_options.read_batch_size = core_options.GetReadBatchSize();
+ impl->read_options.adaptive_prefetch_strategy = core_options.EnableAdaptivePrefetchStrategy();
+ impl->read_options.prefetch_io_metrics_enabled = core_options.PrefetchIoMetricsEnabled();
+ impl->executor = executor;
+ if (!partition_keys.empty()) {
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BinaryRowPartitionComputer> partition_computer,
+ BinaryRowPartitionComputer::Create(
+ partition_keys, table_schema, table->PartitionDefaultName(),
+ core_options.LegacyPartitionNameEnabled(), memory_pool));
+ impl->partition_computer = std::move(partition_computer);
+ }
+
+ return std::unique_ptr<FormatTableRead>(new FormatTableRead(std::move(impl)));
+}
+
+Result<std::unique_ptr<BatchReader>> FormatTableRead::CreateSplitReader(
+ const std::shared_ptr<Split>& split) {
+ auto format_split = std::dynamic_pointer_cast<FormatDataSplit>(split);
+ if (format_split == nullptr) {
+ return Status::Invalid("format table read only accepts a FormatDataSplit");
+ }
+
+ // A `Split` the caller held on to may have been planned from another table or before these
+ // files moved, so whether a file belongs to this table is asked rather than trusted.
+ PAIMON_RETURN_NOT_OK(FormatPathValidation::ValidatePartitionKeys(
+ impl_->table, format_split->partition, "split"));
+ for (const FormatDataSplit::FileMeta& file : format_split->files) {
+ PAIMON_RETURN_NOT_OK(FormatPathValidation::ValidatePathUnderLocation(
+ file.file_path, impl_->table->Location(), "split"));
+ // A split mixing partitions would read rows back under values they never had.
+ PAIMON_RETURN_NOT_OK(FormatPathValidation::ValidateFileInPartition(
+ impl_->table, file.file_path, format_split->partition, "split"));
+ PAIMON_RETURN_NOT_OK(
+ FormatPathValidation::ValidateFileIsVisible(impl_->table, file.file_path, "split"));
+ if (file.file_size < 0) {
+ return Status::Invalid(fmt::format("split gives {} a negative size", file.file_path));
+ }
+ }
+
+ // The partition values in the shape a `FieldMappingReader` reads them from; a directory named
+ // after the default partition name reads back as null.
+ BinaryRow partition = BinaryRow::EmptyRow();
+ if (impl_->partition_computer != nullptr) {
+ PAIMON_ASSIGN_OR_RAISE(partition,
+ impl_->partition_computer->ToBinaryRow(format_split->partition));
+ }
+
+ // One reader builder serves the whole split, built by the component the managed table path
+ // uses, so a format table's file is read with the cache and read hints any other data file is.
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<ReaderBuilder> builder,
+ DataFileReaderFactory::CreateReaderBuilder(
+ impl_->format_identifier, impl_->table->Options(),
+ /*extra_format_options=*/{}, impl_->read_options, impl_->pool));
+ std::shared_ptr<ReaderBuilder> reader_builder(std::move(builder));
+
+ // Captured by value, so a file's reader outlives this `FormatTableRead`.
+ std::shared_ptr<FormatTable> table = impl_->table;
+ std::shared_ptr<const std::vector<DataField>> table_fields = impl_->table_fields;
+ std::shared_ptr<const std::set<std::string>> read_file_columns = impl_->read_file_columns;
+ std::shared_ptr<FieldMappingBuilder> field_mapping_builder = impl_->field_mapping_builder;
+ std::shared_ptr<MemoryPool> pool = impl_->pool;
+ std::shared_ptr<arrow::MemoryPool> arrow_pool = impl_->arrow_pool;
+ std::shared_ptr<Executor> executor = impl_->executor;
+ std::string format_identifier = impl_->format_identifier;
+ DataFileReadOptions read_options = impl_->read_options;
+
+ // Each file is named alongside its factory, so every failure says which file it was.
+ std::vector<LazyConcatBatchReader::Source> sources;
+ sources.reserve(format_split->files.size());
+ for (const FormatDataSplit::FileMeta& file : format_split->files) {
+ LazyConcatBatchReader::Source source;
+ source.name = file.file_path;
+ source.open = [table, reader_builder, table_fields, read_file_columns,
+ field_mapping_builder, partition, pool, arrow_pool, executor,
+ format_identifier, read_options,
+ file]() -> Result<std::unique_ptr<BatchReader>> {
+ // `Open` trusts the length it is handed, and a stale one would truncate an
+ // object-store read or send it past the end, so the file system is asked for it.
+ //
+ // TODO(SteNicholas): open straight from the size the plan carries once a split can
+ // vouch for it. StarRocks and DuckDB both found this extra stat a hotspot on object
+ // stores with small files, and #189 removed it from the managed table path. What is
+ // missing here is a way to say a `FormatDataSplit` reached the read unedited.
+ PAIMON_ASSIGN_OR_RAISE(FileStatus status,
+ table->GetFileSystem()->GetFileStatus(file.file_path));
+ if (status.IsDir()) {
+ return Status::Invalid("the split names a directory, not a data file");
+ }
+ if (file.file_size != status.GetLen()) {
+ return Status::Invalid(fmt::format(
+ "the split says it is {} bytes but it is {}; the plan was made against a "
+ "different version of the file",
+ file.file_size, status.GetLen()));
+ }
+ PAIMON_ASSIGN_OR_RAISE(
+ std::unique_ptr<FileBatchReader> file_reader,
+ DataFileReaderFactory::Open(format_identifier, file.file_path, status.GetLen(),
+ reader_builder.get(), read_options,
+ table->GetFileSystem(), executor, pool, arrow_pool));
+
+ // Built per file, against the columns that file holds rather than the table's. See
+ // `DescribeFileColumns()`.
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> c_file_schema,
+ file_reader->GetFileSchema());
+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> file_schema,
+ arrow::ImportSchema(c_file_schema.get()));
+ PAIMON_ASSIGN_OR_RAISE(
+ std::vector<DataField> file_fields,
+ DescribeFileColumns(file_schema, *table_fields, table->PartitionKeys(),
+ *read_file_columns, table->GetFormat(), file.file_path));
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FieldMapping> field_mapping,
+ field_mapping_builder->CreateFieldMapping(file_fields));
+ std::shared_ptr<arrow::Schema> file_read_schema =
+ DataField::ConvertDataFieldsToArrowSchema(
+ field_mapping->non_partition_info.non_partition_data_schema);
+ std::shared_ptr<Predicate> pushdown_predicate =
+ field_mapping->non_partition_info.non_partition_filter;
+
+ ::ArrowSchema c_read_schema;
+ ArrowSchemaMarkReleased(&c_read_schema);
+ ScopeGuard read_schema_guard(
+ [&c_read_schema]() { ArrowSchemaRelease(&c_read_schema); });
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*file_read_schema, &c_read_schema));
+ PAIMON_RETURN_NOT_OK(file_reader->SetReadSchema(&c_read_schema, pushdown_predicate,
+ /*selection_bitmap=*/std::nullopt));
+
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FieldMappingReader> reader,
+ FieldMappingReader::Create(
+ field_mapping_builder->GetReadFieldCount(),
+ std::move(file_reader), partition, std::move(field_mapping),
+ /*skip_map_selected_keys_filter_field_ids=*/{}, arrow_pool));
+ return std::unique_ptr<BatchReader>(std::move(reader));
+ };
+ sources.push_back(std::move(source));
+ }
+
+ return std::make_unique<LazyConcatBatchReader>(std::move(sources), impl_->arrow_pool);
+}
+
+Result<std::unique_ptr<BatchReader>> FormatTableRead::ApplyFilterAndRowKind(
+ std::unique_ptr<BatchReader>&& reader) {
+ std::unique_ptr<BatchReader> result = std::move(reader);
+ if (impl_->filter_predicate != nullptr) {
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<PredicateBatchReader> filtered,
+ PredicateBatchReader::Create(
+ std::move(result), impl_->filter_predicate, impl_->arrow_pool));
+ result = std::move(filtered);
+ }
+ // Every row is an insert, but `BatchReader::NextBatch()` still promises the leading
+ // `_VALUE_KIND` field: an engine reading by field index would find its columns shifted.
+ return std::make_unique<CompleteRowKindBatchReader>(std::move(result), impl_->arrow_pool);
+}
+
+Result<std::unique_ptr<BatchReader>> FormatTableRead::CreateReader(
+ const std::shared_ptr<Split>& split) {
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> reader, CreateSplitReader(split));
+ return ApplyFilterAndRowKind(std::move(reader));
+}
+
+Result<std::unique_ptr<BatchReader>> FormatTableRead::CreateReader(
+ const std::vector<std::shared_ptr<Split>>& splits) {
+ std::vector<std::unique_ptr<BatchReader>> split_readers;
+ split_readers.reserve(splits.size());
+ for (const std::shared_ptr<Split>& split : splits) {
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> reader, CreateSplitReader(split));
+ split_readers.push_back(std::move(reader));
+ }
+ std::unique_ptr<BatchReader> reader =
+ std::make_unique<ConcatBatchReader>(std::move(split_readers), impl_->arrow_pool);
+ return ApplyFilterAndRowKind(std::move(reader));
+}
+
+} // namespace paimon
diff --git a/src/paimon/core/table/format/format_table_read.h b/src/paimon/core/table/format/format_table_read.h
new file mode 100644
index 00000000..edfbbe91
--- /dev/null
+++ b/src/paimon/core/table/format/format_table_read.h
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "paimon/common/reader/data_file_reader_factory.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/read_context.h"
+#include "paimon/reader/batch_reader.h"
+#include "paimon/result.h"
+#include "paimon/table/format/format_table.h"
+#include "paimon/table/source/split.h"
+#include "paimon/table/source/table_read.h"
+
+namespace paimon {
+
+class Executor;
+class Predicate;
+
+/// Reads the splits a `FormatTableScan` produced.
+///
+/// Partition columns are filled from the split rather than read: the file is asked only for the
+/// columns that are not partition keys, so the directory decides a file's partition values even
+/// when the file stores them too. Batches carry the leading `_VALUE_KIND` field every
+/// `BatchReader` promises, and borrow memory from the reader that produced them.
+///
+/// A read may be shared between threads; the `BatchReader`s it hands out may not be.
+class FormatTableRead : public TableRead {
+ public:
+ /// Reads what a `ReadContext` asks for, which is how `TableRead::Create()` reaches a format
+ /// table. Everything - columns, predicate, pool, executor and how a file is opened - comes
+ /// from the context. A setting a format table cannot honour is refused by name rather than
+ /// dropped: a projected read schema, and a real-time context.
+ static Result<std::unique_ptr<FormatTableRead>> Create(
+ const std::shared_ptr<FormatTable>& table,
+ const std::shared_ptr<ReadContext>& read_context);
+
+ /// Names the columns, the predicate and the pool directly, without a `ReadContext`. Files are
+ /// opened plainly: no prefetch, no cache. For tests; production goes through `Create()`.
+ ///
+ /// @param projection Columns to read, in the order they should appear; absent reads them all.
+ /// @param predicate Pushed into the file readers, so on its own it is best effort. Validated
+ /// as on the managed table path: every field it names must be one the projection keeps.
+ /// @param enable_predicate_filter Whether the returned reader also applies `predicate`
+ /// exactly. Off by default, as everywhere else in paimon-cpp.
+ static Result<std::unique_ptr<FormatTableRead>> TEST_Create(
+ const std::shared_ptr<FormatTable>& table,
+ const std::optional<std::vector<std::string>>& projection,
+ const std::shared_ptr<MemoryPool>& pool, const std::shared_ptr<Predicate>& predicate,
+ bool enable_predicate_filter);
+
+ ~FormatTableRead() override;
+
+ /// Reads one split's files, in the split's order.
+ Result<std::unique_ptr<BatchReader>> CreateReader(const std::shared_ptr<Split>& split) override;
+
+ /// Reads several splits, in the given order.
+ Result<std::unique_ptr<BatchReader>> CreateReader(
+ const std::vector<std::shared_ptr<Split>>& splits) override;
+
+ class Impl;
+
+ private:
+ explicit FormatTableRead(std::unique_ptr<Impl> impl);
+
+ /// Shared body of `Create()` and `TEST_Create()`. Every setting arrives resolved, so each has
+ /// exactly one source and no `ReadContext` is read here as well.
+ ///
+ /// @param read_options What a data file is opened with, as far as the caller decides it; a
+ /// default-constructed value opens files plainly. The fields describing the table
+ /// rather than the read are filled in from its options afterwards.
+ /// @param executor Runs a prefetching reader's read-ahead. Null when nothing asked for it.
+ static Result<std::unique_ptr<FormatTableRead>> CreateInternal(
+ const std::shared_ptr<FormatTable>& table,
+ const std::optional<std::vector<std::string>>& projection,
+ const std::shared_ptr<MemoryPool>& pool, const std::shared_ptr<Predicate>& predicate,
+ bool enable_predicate_filter, const DataFileReadOptions& read_options,
+ const std::shared_ptr<Executor>& executor);
+
+ /// Builds the reader over one split's files, without the predicate filter or the row kinds.
+ Result<std::unique_ptr<BatchReader>> CreateSplitReader(const std::shared_ptr<Split>& split);
+
+ /// Wraps a reader in the exact predicate filter when one was asked for, and then in the
+ /// `_VALUE_KIND` field. The filter runs underneath it, on the table's own columns.
+ Result<std::unique_ptr<BatchReader>> ApplyFilterAndRowKind(
+ std::unique_ptr<BatchReader>&& reader);
+
+ std::unique_ptr<Impl> impl_;
+};
+
+} // namespace paimon
diff --git a/src/paimon/core/table/format/format_table_scan.cpp b/src/paimon/core/table/format/format_table_scan.cpp
new file mode 100644
index 00000000..ce46d123
--- /dev/null
+++ b/src/paimon/core/table/format/format_table_scan.cpp
@@ -0,0 +1,288 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/core/table/format/format_table_scan.h"
+
+#include <algorithm>
+#include <memory>
+#include <string>
+#include <utility>
+
+#include "arrow/c/bridge.h"
+#include "fmt/format.h"
+#include "fmt/ranges.h"
+#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/bin_packing.h"
+#include "paimon/common/utils/path_util.h"
+#include "paimon/core/core_options.h"
+#include "paimon/core/table/format/format_data_split.h"
+#include "paimon/core/table/format/format_file_listing.h"
+#include "paimon/core/table/format/format_path_validation.h"
+#include "paimon/core/table/source/plan_impl.h"
+#include "paimon/core/utils/partition_path_utils.h"
+#include "paimon/fs/file_system.h"
+#include "paimon/logging.h"
+#include "paimon/memory/memory_pool.h"
+
+namespace paimon {
+
+namespace {
+Logger* ScanLogger() {
+ static std::unique_ptr<Logger> logger = Logger::GetLogger("FormatTableScan");
+ return logger.get();
+}
+} // namespace
+
+FormatTableScan::FormatTableScan(const std::shared_ptr<FormatTable>& table,
+ const std::map<std::string, std::string>& partition_filter,
+ std::map<std::string, BinaryRow> partition_filter_values,
+ const std::optional<int32_t>& limit, int64_t target_split_size,
+ int64_t open_file_cost,
+ std::unique_ptr<BinaryRowPartitionComputer> partition_computer)
+ : table_(table),
+ partition_filter_(partition_filter),
+ partition_filter_values_(std::move(partition_filter_values)),
+ limit_(limit),
+ target_split_size_(target_split_size),
+ open_file_cost_(open_file_cost),
+ partition_computer_(std::move(partition_computer)) {}
+
+FormatTableScan::~FormatTableScan() = default;
+
+Result<std::unique_ptr<FormatTableScan>> FormatTableScan::Create(
+ const std::shared_ptr<FormatTable>& table,
+ const std::map<std::string, std::string>& partition_filter,
+ const std::optional<int32_t>& limit) {
+ if (table == nullptr) {
+ return Status::Invalid("format table scan requires a table");
+ }
+ const std::vector<std::string>& partition_keys = table->PartitionKeys();
+ for (const auto& filter : partition_filter) {
+ const std::string& key = filter.first;
+ if (std::find(partition_keys.begin(), partition_keys.end(), key) == partition_keys.end()) {
+ return Status::Invalid(
+ fmt::format("partition filter field '{}' is not a partition key of table {}", key,
+ table->FullName()));
+ }
+ }
+ // Through `CoreOptions`, so `"128 mb"` means what it does elsewhere.
+ PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options,
+ CoreOptions::FromMap(table->Options(), table->GetFileSystem()));
+
+ // Each value is read into its column's type, so a partition is matched by what the column
+ // holds rather than by the text either side spelled it with.
+ std::unique_ptr<BinaryRowPartitionComputer> partition_computer;
+ std::map<std::string, BinaryRow> partition_filter_values;
+ if (!partition_keys.empty()) {
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> c_schema, table->GetArrowSchema());
+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> table_schema,
+ arrow::ImportSchema(c_schema.get()));
+ PAIMON_ASSIGN_OR_RAISE(partition_computer,
+ BinaryRowPartitionComputer::Create(
+ partition_keys, table_schema, table->PartitionDefaultName(),
+ core_options.LegacyPartitionNameEnabled(), GetDefaultPool()));
+ for (const auto& [key, value] : partition_filter) {
+ // One key at a time: discovery tests one directory level at a time, and only a row of
+ // the same shape can be compared against it.
+ Result<BinaryRow> filter_value = partition_computer->ToPartialBinaryRow({{key, value}});
+ if (!filter_value.ok()) {
+ // A value no partition of this table could hold is the caller's mistake, not an
+ // empty result: matching nothing would look like a table with no such rows.
+ return Status::Invalid(
+ fmt::format("partition filter {} cannot be read into the partition columns of "
+ "table {}: {}",
+ fmt::format("{}", partition_filter), table->FullName(),
+ filter_value.status().message()));
+ }
+ partition_filter_values.emplace(key, std::move(filter_value).value());
+ }
+ }
+ return std::unique_ptr<FormatTableScan>(new FormatTableScan(
+ table, partition_filter, std::move(partition_filter_values), limit,
+ core_options.GetSourceSplitTargetSize(), core_options.GetSourceSplitOpenFileCost(),
+ std::move(partition_computer)));
+}
+
+bool FormatTableScan::PartitionValuePassesFilter(const std::string& partition_key,
+ const std::string& value) const {
+ auto filter_iter = partition_filter_values_.find(partition_key);
+ if (filter_iter == partition_filter_values_.end()) {
+ return true;
+ }
+ // `partition_filter_values_` is empty unless the table is partitioned, so the computer is
+ // there. Both sides are built the same way, so equal values give equal rows.
+ Result<BinaryRow> directory_value =
+ partition_computer_->ToPartialBinaryRow({{partition_key, value}});
+ if (!directory_value.ok()) {
+ PAIMON_LOG_DEBUG(ScanLogger(),
+ "Skipping directory of format table %s: '%s' is not a value partition "
+ "column '%s' can hold: %s",
+ table_->FullName().c_str(), value.c_str(), partition_key.c_str(),
+ directory_value.status().ToString().c_str());
+ return false;
+ }
+ return directory_value.value() == filter_iter->second;
+}
+
+Result<std::vector<FormatTableScan::PartitionAndPath>> FormatTableScan::FindPartitions() const {
+ const std::vector<std::string>& partition_keys = table_->PartitionKeys();
+ PAIMON_LOG_DEBUG(ScanLogger(), "Finding partitions for format table %s, partition filter: %s",
+ table_->FullName().c_str(), fmt::format("{}", partition_filter_).c_str());
+ std::shared_ptr<FileSystem> file_system = table_->GetFileSystem();
+
+ const bool only_value = table_->PartitionOnlyValueInPath();
+ // The one hidden name that is table content: a null partition's directory in the value-only
+ // layout.
+ const std::string& default_partition_name = table_->PartitionDefaultName();
+
+ std::vector<PartitionAndPath> level = {
+ {std::map<std::string, std::string>(), table_->Location()}};
+ bool at_table_location = true;
+ for (const std::string& partition_key : partition_keys) {
+ std::vector<PartitionAndPath> next;
+ for (const auto& [partition, directory] : level) {
+ std::vector<BasicFileStatus> children;
+ Status status = file_system->ListDir(directory, &children);
+ if (status.IsNotExist()) {
+ // Gone since it was listed, or a table directory not created yet; either way
+ // the rest of the listing stands.
+ continue;
+ }
+ PAIMON_RETURN_NOT_OK(status);
+ for (const BasicFileStatus& child : children) {
+ if (!child.IsDir()) {
+ continue;
+ }
+ std::string name = PathUtil::GetName(child.GetPath());
+ if (PartitionPathUtils::IsHiddenName(name) &&
+ !(only_value && name == default_partition_name)) {
+ continue;
+ }
+ if (at_table_location && table_->LocationCarriesPaimonMetadata() &&
+ FormatFileListing::IsReservedDirectory(name)) {
+ continue;
+ }
+ std::string value;
+ if (only_value) {
+ // Nothing but the level says which key a directory belongs to, so every
+ // directory here is one.
+ value = PartitionPathUtils::UnescapePathName(name);
+ } else {
+ std::optional<std::pair<std::string, std::string>> key_value =
+ PartitionPathUtils::ExtractPartitionKeyValue(name);
+ if (!key_value || key_value->first != partition_key) {
+ // Something else lives here: another layout, or a nested table.
+ continue;
+ }
+ value = std::move(key_value->second);
+ }
+ if (!PartitionValuePassesFilter(partition_key, value)) {
+ continue;
+ }
+ // The directory's own value, not the normalised one: a format table's
+ // partitions are its directories, so a plan reports what is on disk.
+ std::map<std::string, std::string> child_partition = partition;
+ child_partition[partition_key] = std::move(value);
+ next.emplace_back(std::move(child_partition), child.GetPath());
+ }
+ }
+ level = std::move(next);
+ at_table_location = false;
+ }
+ PAIMON_LOG_DEBUG(ScanLogger(), "Found %zu partitions of format table %s", level.size(),
+ table_->FullName().c_str());
+ return level;
+}
+
+Result<std::vector<std::shared_ptr<Split>>> FormatTableScan::CreateSplits(
+ const std::string& directory, const std::map<std::string, std::string>& partition) const {
+ // `directory` is a complete partition, or the table itself when unpartitioned, so no partition
+ // directory is left below it and `partition_levels` stays 0.
+ FormatDataFileListingOptions listing;
+ // Only right at the table's location are `schema` and `branch` its metadata.
+ PAIMON_ASSIGN_OR_RAISE(bool at_location,
+ FormatPathValidation::IsTableLocation(table_, directory));
+ listing.skip_reserved_directories = at_location && table_->LocationCarriesPaimonMetadata();
+ std::vector<FormatDataSplit::FileMeta> files;
+ PAIMON_RETURN_NOT_OK(
+ FormatFileListing::ListDataFiles(table_->GetFileSystem(), directory, listing, &files));
+ std::vector<std::shared_ptr<Split>> splits;
+ if (files.empty()) {
+ return splits;
+ }
+ // The file system promises no listing order, and a scan's output should be reproducible.
+ std::sort(files.begin(), files.end(),
+ [](const FormatDataSplit::FileMeta& left, const FormatDataSplit::FileMeta& right) {
+ return left.file_path < right.file_path;
+ });
+ // A file is never cut up, and each entry costs at least the open-file cost, so a split
+ // cannot gather so many small files that opening them outweighs reading them.
+ const int64_t open_file_cost = open_file_cost_;
+ std::vector<std::vector<FormatDataSplit::FileMeta>> bins =
+ BinPacking::PackForOrdered<FormatDataSplit::FileMeta>(
+ std::move(files),
+ [open_file_cost](const FormatDataSplit::FileMeta& file) {
+ return std::max(file.file_size, open_file_cost);
+ },
+ target_split_size_);
+ splits.reserve(bins.size());
+ for (const std::vector<FormatDataSplit::FileMeta>& bin : bins) {
+ splits.push_back(std::make_shared<FormatDataSplit>(bin, partition));
+ }
+ return splits;
+}
+
+Result<std::shared_ptr<Plan>> FormatTableScan::CreatePlan() {
+ std::vector<std::shared_ptr<Split>> splits;
+ if (limit_ && *limit_ <= 0) {
+ return std::make_shared<PlanImpl>(std::nullopt, splits);
+ }
+
+ if (table_->PartitionKeys().empty()) {
+ PAIMON_ASSIGN_OR_RAISE(splits, CreateSplits(table_->Location(), {}));
+ return std::make_shared<PlanImpl>(std::nullopt, splits);
+ }
+
+ PAIMON_ASSIGN_OR_RAISE(std::vector<PartitionAndPath> partitions, FindPartitions());
+ std::sort(partitions.begin(), partitions.end(),
+ [](const auto& left, const auto& right) { return left.second < right.second; });
+ for (const auto& [partition, directory] : partitions) {
+ PAIMON_ASSIGN_OR_RAISE(std::vector<std::shared_ptr<Split>> partition_splits,
+ CreateSplits(directory, partition));
+ splits.insert(splits.end(), std::make_move_iterator(partition_splits.begin()),
+ std::make_move_iterator(partition_splits.end()));
+ }
+ return std::make_shared<PlanImpl>(std::nullopt, splits);
+}
+
+Result<std::vector<std::map<std::string, std::string>>> FormatTableScan::ListPartitions() const {
+ std::vector<std::map<std::string, std::string>> result;
+ if (table_->PartitionKeys().empty()) {
+ return result;
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::vector<PartitionAndPath> partitions, FindPartitions());
+ std::sort(partitions.begin(), partitions.end(),
+ [](const auto& left, const auto& right) { return left.second < right.second; });
+ result.reserve(partitions.size());
+ for (PartitionAndPath& partition_and_path : partitions) {
+ result.push_back(std::move(partition_and_path.first));
+ }
+ return result;
+}
+
+} // namespace paimon
diff --git a/src/paimon/core/table/format/format_table_scan.h b/src/paimon/core/table/format/format_table_scan.h
new file mode 100644
index 00000000..7ac2dccf
--- /dev/null
+++ b/src/paimon/core/table/format/format_table_scan.h
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "paimon/common/data/binary_row.h"
+#include "paimon/common/utils/binary_row_partition_computer.h"
+#include "paimon/result.h"
+#include "paimon/table/format/format_table.h"
+#include "paimon/table/source/plan.h"
+#include "paimon/table/source/table_scan.h"
+
+namespace paimon {
+
+/// Plans a read of a format table by listing its directories.
+///
+/// Partitions come from the directory layout and a partition's files are packed into splits of
+/// about `source.split.target-size`; `docs/source/user_guide/format_table.rst` describes the
+/// layout. A name starting with `_` or `.` is skipped and not descended into, being how an engine
+/// marks uncommitted output, with `partition.default-name` the one exception.
+///
+/// Planning leaves the scan as it was, so one may be shared between threads.
+class FormatTableScan : public TableScan {
+ public:
+ /// A partition's values paired with the directory they were read from.
+ using PartitionAndPath = std::pair<std::map<std::string, std::string>, std::string>;
+
+ /// @param partition_filter Partition values to keep, keyed by partition field name. A key
+ /// absent from the map is unconstrained; an empty map keeps every partition. A value is
+ /// matched in its column type rather than as text, so `1` and `01` name the same `INT`
+ /// partition, and one that will not read into that type is refused rather than
+ /// silently matching nothing.
+ /// @param limit Upper bound on the rows the caller will read. A format table records no row
+ /// counts, so a positive limit drops no split and the caller still has to stop itself.
+ static Result<std::unique_ptr<FormatTableScan>> Create(
+ const std::shared_ptr<FormatTable>& table,
+ const std::map<std::string, std::string>& partition_filter,
+ const std::optional<int32_t>& limit);
+
+ ~FormatTableScan() override;
+
+ /// Plans the read: the splits of every partition that passes the filter, in a stable order.
+ ///
+ /// A missing table directory is answered differently by the two shapes of table: an
+ /// unpartitioned one has only its location to list, so a location that is not there fails -
+ /// the only way to tell a wrong location from a table with no rows - while a partitioned one
+ /// finds no partitions below it and plans nothing.
+ Result<std::shared_ptr<Plan>> CreatePlan() override;
+
+ /// Lists the partitions the table's directory layout holds, whether or not they hold data,
+ /// in a stable order. Every partition key gets a value; a directory named
+ /// `partition.default-name` reads back as that name, standing for a null partition value.
+ Result<std::vector<std::map<std::string, std::string>>> ListPartitions() const override;
+
+ private:
+ FormatTableScan(const std::shared_ptr<FormatTable>& table,
+ const std::map<std::string, std::string>& partition_filter,
+ std::map<std::string, BinaryRow> partition_filter_values,
+ const std::optional<int32_t>& limit, int64_t target_split_size,
+ int64_t open_file_cost,
+ std::unique_ptr<BinaryRowPartitionComputer> partition_computer);
+
+ /// Whether the directory value `value` of `partition_key` passes the filter.
+ ///
+ /// Both sides are compared in the column's type rather than as text, so a blank value and the
+ /// null partition stay apart even though a directory name cannot tell them apart. A value that
+ /// will not read into that type is skipped.
+ bool PartitionValuePassesFilter(const std::string& partition_key,
+ const std::string& value) const;
+
+ /// Lists partition directories under the table location that pass the partition filter, paired
+ /// with the partition values their names spell out.
+ Result<std::vector<PartitionAndPath>> FindPartitions() const;
+
+ /// Collects the data files under `directory` and everything below it and packs them into
+ /// splits of about the target size. Empty when there is no data file.
+ Result<std::vector<std::shared_ptr<Split>>> CreateSplits(
+ const std::string& directory, const std::map<std::string, std::string>& partition) const;
+
+ std::shared_ptr<FormatTable> table_;
+ /// The filter as the caller gave it, which is what an error or a log line reports.
+ std::map<std::string, std::string> partition_filter_;
+ /// The same filter as the table's own types hold it, one single-field row per key. Empty when
+ /// the table is not partitioned or nothing was filtered on.
+ std::map<std::string, BinaryRow> partition_filter_values_;
+ std::optional<int32_t> limit_;
+ int64_t target_split_size_;
+ int64_t open_file_cost_;
+ /// Reads a partition value into its column type and renders it back out. Null when the table
+ /// is not partitioned, which leaves nothing to filter on.
+ std::unique_ptr<BinaryRowPartitionComputer> partition_computer_;
+};
+
+} // namespace paimon
diff --git a/src/paimon/core/table/format/format_table_test.cpp b/src/paimon/core/table/format/format_table_test.cpp
new file mode 100644
index 00000000..f4d86cb8
--- /dev/null
+++ b/src/paimon/core/table/format/format_table_test.cpp
@@ -0,0 +1,5060 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/table/format/format_table.h"
+
+#include <algorithm>
+#include <filesystem>
+#include <limits>
+#include <map>
+#include <memory>
+#include <optional>
+#include <string>
+#include <system_error>
+#include <tuple>
+#include <utility>
+#include <vector>
+
+#include "arrow/api.h"
+#include "arrow/c/bridge.h"
+#include "gtest/gtest.h"
+#include "paimon/cache/cache.h"
+#include "paimon/commit_context.h"
+#include "paimon/commit_message.h"
+#include "paimon/common/table/special_fields.h"
+#include "paimon/common/types/row_kind.h"
+#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/common/utils/path_util.h"
+#include "paimon/common/utils/string_utils.h"
+#include "paimon/core/schema/schema_manager.h"
+#include "paimon/core/schema/table_schema.h"
+#include "paimon/core/table/format/format_commit_message.h"
+#include "paimon/core/table/format/format_data_split.h"
+#include "paimon/core/table/format/format_path_validation.h"
+#include "paimon/core/table/format/format_table_commit.h"
+#include "paimon/core/table/format/format_table_read.h"
+#include "paimon/core/table/format/format_table_scan.h"
+#include "paimon/core/table/format/format_table_write.h"
+#include "paimon/defs.h"
+#include "paimon/file_store_commit.h"
+#include "paimon/file_store_write.h"
+#include "paimon/fs/file_system.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/predicate/literal.h"
+#include "paimon/predicate/predicate_builder.h"
+#include "paimon/read_context.h"
+#include "paimon/record_batch.h"
+#include "paimon/scan_context.h"
+#include "paimon/status.h"
+#include "paimon/table/source/split.h"
+#include "paimon/table/source/table_read.h"
+#include "paimon/table/source/table_scan.h"
+#include "paimon/testing/utils/testharness.h"
+#include "paimon/write_context.h"
+
+namespace paimon::test {
+
+namespace {
+
+/// What `ListPartitions()` returns. Aliased because a macro argument cannot hold the comma in
+/// `std::map<std::string, std::string>`: the preprocessor would read it as two arguments.
+using PartitionList = std::vector<std::map<std::string, std::string>>;
+
+std::shared_ptr<arrow::Schema> MakeSchema() {
+ return arrow::schema({arrow::field("id", arrow::int32()), arrow::field("name", arrow::utf8()),
+ arrow::field("dt", arrow::utf8())});
+}
+
+/// Wraps a file system and writes down what a write did to each path, in order. Only the order
+/// gives away a temp file deleted while its stream is still open, which on a store that flushes
+/// from the destructor lands the write after the delete.
+class CallOrderFileSystem : public FileSystem {
+ public:
+ explicit CallOrderFileSystem(const std::shared_ptr<FileSystem>& delegate)
+ : delegate_(delegate), calls_(std::make_shared<std::vector<std::string>>()) {}
+
+ /// What happened, as "<verb> <path>" in the order it happened.
+ const std::vector<std::string>& Calls() const {
+ return *calls_;
+ }
+
+ using FileSystem::Open;
+
+ Result<std::unique_ptr<InputStream>> Open(const std::string& path) const override {
+ return delegate_->Open(path);
+ }
+ Result<std::unique_ptr<OutputStream>> Create(const std::string& path,
+ bool overwrite) const override {
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<OutputStream> out,
+ delegate_->Create(path, overwrite));
+ calls_->push_back("create " + path);
+ return std::unique_ptr<OutputStream>(
+ new RecordingOutputStream(std::move(out), path, calls_));
+ }
+ Status Mkdirs(const std::string& path) const override {
+ return delegate_->Mkdirs(path);
+ }
+ Status Rename(const std::string& src, const std::string& dst) const override {
+ return delegate_->Rename(src, dst);
+ }
+ Status Delete(const std::string& path, bool recursive = true) const override {
+ calls_->push_back("delete " + path);
+ return delegate_->Delete(path, recursive);
+ }
+ Result<FileStatus> GetFileStatus(const std::string& path) const override {
+ return delegate_->GetFileStatus(path);
+ }
+ Status ListDir(const std::string& directory,
+ std::vector<BasicFileStatus>* status_list) const override {
+ return delegate_->ListDir(directory, status_list);
+ }
+ Status ListFileStatus(const std::string& path,
+ std::vector<FileStatus>* status_list) const override {
+ return delegate_->ListFileStatus(path, status_list);
+ }
+ Result<bool> Exists(const std::string& path) const override {
+ return delegate_->Exists(path);
+ }
+
+ private:
+ /// Records its own close, so a stream still open when its file was deleted can be told apart
+ /// from one that was closed first.
+ class RecordingOutputStream : public OutputStream {
+ public:
+ RecordingOutputStream(std::unique_ptr<OutputStream> delegate, const std::string& path,
+ const std::shared_ptr<std::vector<std::string>>& calls)
+ : delegate_(std::move(delegate)), path_(path), calls_(calls) {}
+
+ Result<int64_t> Write(const char* buffer, int64_t size) override {
+ return delegate_->Write(buffer, size);
+ }
+ Status Flush() override {
+ return delegate_->Flush();
+ }
+ Result<int64_t> GetPos() const override {
+ return delegate_->GetPos();
+ }
+ Result<std::string> GetUri() const override {
+ return delegate_->GetUri();
+ }
+ Status Close() override {
+ calls_->push_back("close " + path_);
+ return delegate_->Close();
+ }
+
+ private:
+ std::unique_ptr<OutputStream> delegate_;
+ std::string path_;
+ std::shared_ptr<std::vector<std::string>> calls_;
+ };
+
+ std::shared_ptr<FileSystem> delegate_;
+ /// Shared with every stream this hands out, so one list holds the whole sequence.
+ std::shared_ptr<std::vector<std::string>> calls_;
+};
+
+/// When a `FailingRenameFileSystem` gives up on the rename it is told to fail.
+enum class RenameFailure {
+ /// Nothing moves, so the target is not there when the commit gives up.
+ kWithoutPublishing,
+ /// The rename takes effect and reports a failure all the same - what a store that completed
+ /// the request and lost its response looks like from here.
+ kAfterPublishing,
+};
+
+/// Wraps a file system and refuses the nth `Rename()`, so that a commit can be stopped part way
+/// through publishing - after some files are visible and, for an overwrite, after the data they
+/// replace is already gone.
+class FailingRenameFileSystem : public FileSystem {
+ public:
+ FailingRenameFileSystem(const std::shared_ptr<FileSystem>& delegate, int32_t fail_nth_rename,
+ RenameFailure failure = RenameFailure::kWithoutPublishing)
+ : delegate_(delegate), fail_nth_rename_(fail_nth_rename), failure_(failure) {}
+
+ using FileSystem::Open;
+
+ Result<std::unique_ptr<InputStream>> Open(const std::string& path) const override {
+ return delegate_->Open(path);
+ }
+ Result<std::unique_ptr<OutputStream>> Create(const std::string& path,
+ bool overwrite) const override {
+ return delegate_->Create(path, overwrite);
+ }
+ Status Mkdirs(const std::string& path) const override {
+ return delegate_->Mkdirs(path);
+ }
+ Status Rename(const std::string& src, const std::string& dst) const override {
+ if (++renames_ != fail_nth_rename_) {
+ return delegate_->Rename(src, dst);
+ }
+ if (failure_ == RenameFailure::kAfterPublishing) {
+ PAIMON_RETURN_NOT_OK(delegate_->Rename(src, dst));
+ }
+ return Status::IOError("injected rename failure");
+ }
+ Status Delete(const std::string& path, bool recursive = true) const override {
+ return delegate_->Delete(path, recursive);
+ }
+ Result<FileStatus> GetFileStatus(const std::string& path) const override {
+ return delegate_->GetFileStatus(path);
+ }
+ Status ListDir(const std::string& directory,
+ std::vector<BasicFileStatus>* status_list) const override {
+ return delegate_->ListDir(directory, status_list);
+ }
+ Status ListFileStatus(const std::string& path,
+ std::vector<FileStatus>* status_list) const override {
+ return delegate_->ListFileStatus(path, status_list);
+ }
+ Result<bool> Exists(const std::string& path) const override {
+ return delegate_->Exists(path);
+ }
+
+ private:
+ std::shared_ptr<FileSystem> delegate_;
+ int32_t fail_nth_rename_;
+ RenameFailure failure_;
+ mutable int32_t renames_ = 0;
+};
+
+/// The one call a `FailingWriteFileSystem`'s streams refuse.
+enum class FailingStreamCall { kGetPos, kFlush, kClose };
+
+/// Wraps a file system and hands out streams that fail one call, so that a write can be stopped
+/// where a real store would stop it: after the writer is finished and while the file it produced
+/// is still hidden.
+class FailingWriteFileSystem : public FileSystem {
+ public:
+ /// `armed` says whether the streams refuse from the start. A test that has to reach past the
+ /// writer's own construction - a parquet writer asks the stream where it is as it opens the
+ /// file - hands out `false` and calls `Arm()` once a file is open and written to.
+ FailingWriteFileSystem(const std::shared_ptr<FileSystem>& delegate, FailingStreamCall failing,
+ bool armed = true)
+ : delegate_(delegate), failing_(failing), armed_(std::make_shared<bool>(armed)) {}
+
+ /// Refuses from here on, through the stream already open as well as any opened later.
+ void Arm() const {
+ *armed_ = true;
+ }
+
+ using FileSystem::Open;
+
+ Result<std::unique_ptr<InputStream>> Open(const std::string& path) const override {
+ return delegate_->Open(path);
+ }
+ Result<std::unique_ptr<OutputStream>> Create(const std::string& path,
+ bool overwrite) const override {
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<OutputStream> out,
+ delegate_->Create(path, overwrite));
+ return std::unique_ptr<OutputStream>(
+ new FailingOutputStream(std::move(out), failing_, armed_));
+ }
+ Status Mkdirs(const std::string& path) const override {
+ return delegate_->Mkdirs(path);
+ }
+ Status Rename(const std::string& src, const std::string& dst) const override {
+ return delegate_->Rename(src, dst);
+ }
+ Status Delete(const std::string& path, bool recursive = true) const override {
+ return delegate_->Delete(path, recursive);
+ }
+ Result<FileStatus> GetFileStatus(const std::string& path) const override {
+ return delegate_->GetFileStatus(path);
+ }
+ Status ListDir(const std::string& directory,
+ std::vector<BasicFileStatus>* status_list) const override {
+ return delegate_->ListDir(directory, status_list);
+ }
+ Status ListFileStatus(const std::string& path,
+ std::vector<FileStatus>* status_list) const override {
+ return delegate_->ListFileStatus(path, status_list);
+ }
+ Result<bool> Exists(const std::string& path) const override {
+ return delegate_->Exists(path);
+ }
+
+ private:
+ class FailingOutputStream : public OutputStream {
+ public:
+ FailingOutputStream(std::unique_ptr<OutputStream> delegate, FailingStreamCall failing,
+ const std::shared_ptr<bool>& armed)
+ : delegate_(std::move(delegate)), failing_(failing), armed_(armed) {}
+
+ Result<int64_t> Write(const char* buffer, int64_t size) override {
+ return delegate_->Write(buffer, size);
+ }
+ Status Flush() override {
+ if (*armed_ && failing_ == FailingStreamCall::kFlush) {
+ return Status::IOError("injected flush failure");
+ }
+ return delegate_->Flush();
+ }
+ Result<int64_t> GetPos() const override {
+ if (*armed_ && failing_ == FailingStreamCall::kGetPos) {
+ return Status::IOError("injected get position failure");
+ }
+ return delegate_->GetPos();
+ }
+ Result<std::string> GetUri() const override {
+ return delegate_->GetUri();
+ }
+ Status Close() override {
+ if (*armed_ && failing_ == FailingStreamCall::kClose) {
+ // Closed all the same, so the file it wrote can still be removed.
+ [[maybe_unused]] Status closed = delegate_->Close();
+ return Status::IOError("injected close failure");
+ }
+ return delegate_->Close();
+ }
+
+ private:
+ std::unique_ptr<OutputStream> delegate_;
+ FailingStreamCall failing_;
+ std::shared_ptr<bool> armed_;
+ };
+
+ std::shared_ptr<FileSystem> delegate_;
+ FailingStreamCall failing_;
+ /// Shared with every stream this hands out, so arming it reaches the one already open.
+ std::shared_ptr<bool> armed_;
+};
+
+/// Wraps a file system and hands out streams that refuse `Write()` from the moment a test arms it.
+///
+/// A format's writer buffers what it is given and writes the file's footer when it is finished, so
+/// arming the refusal between the two is what pins a failure to the finish. Counting writes
+/// instead would pin it to however many a format happens to make first.
+class ArmedWriteFailureFileSystem : public FileSystem {
+ public:
+ explicit ArmedWriteFailureFileSystem(const std::shared_ptr<FileSystem>& delegate)
+ : delegate_(delegate), armed_(std::make_shared<bool>(false)) {}
+
+ /// Every write through this file system fails from here on, including through a stream that
+ /// is already open.
+ void FailFurtherWrites() {
+ *armed_ = true;
+ }
+
+ using FileSystem::Open;
+
+ Result<std::unique_ptr<InputStream>> Open(const std::string& path) const override {
+ return delegate_->Open(path);
+ }
+ Result<std::unique_ptr<OutputStream>> Create(const std::string& path,
+ bool overwrite) const override {
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<OutputStream> out,
+ delegate_->Create(path, overwrite));
+ return std::unique_ptr<OutputStream>(new ArmedOutputStream(std::move(out), armed_));
+ }
+ Status Mkdirs(const std::string& path) const override {
+ return delegate_->Mkdirs(path);
+ }
+ Status Rename(const std::string& src, const std::string& dst) const override {
+ return delegate_->Rename(src, dst);
+ }
+ Status Delete(const std::string& path, bool recursive = true) const override {
+ return delegate_->Delete(path, recursive);
+ }
+ Result<FileStatus> GetFileStatus(const std::string& path) const override {
+ return delegate_->GetFileStatus(path);
+ }
+ Status ListDir(const std::string& directory,
+ std::vector<BasicFileStatus>* status_list) const override {
+ return delegate_->ListDir(directory, status_list);
+ }
+ Status ListFileStatus(const std::string& path,
+ std::vector<FileStatus>* status_list) const override {
+ return delegate_->ListFileStatus(path, status_list);
+ }
+ Result<bool> Exists(const std::string& path) const override {
+ return delegate_->Exists(path);
+ }
+
+ private:
+ class ArmedOutputStream : public OutputStream {
+ public:
+ ArmedOutputStream(std::unique_ptr<OutputStream> delegate,
+ const std::shared_ptr<bool>& armed)
+ : delegate_(std::move(delegate)), armed_(armed) {}
+
+ Result<int64_t> Write(const char* buffer, int64_t size) override {
+ if (*armed_) {
+ return Status::IOError("injected write failure");
+ }
+ return delegate_->Write(buffer, size);
+ }
+ Status Flush() override {
+ return delegate_->Flush();
+ }
+ Result<int64_t> GetPos() const override {
+ return delegate_->GetPos();
+ }
+ Result<std::string> GetUri() const override {
+ return delegate_->GetUri();
+ }
+ Status Close() override {
+ return delegate_->Close();
+ }
+
+ private:
+ std::unique_ptr<OutputStream> delegate_;
+ std::shared_ptr<bool> armed_;
+ };
+
+ std::shared_ptr<FileSystem> delegate_;
+ /// Shared with every stream this hands out, so arming it reaches the one already open.
+ std::shared_ptr<bool> armed_;
+};
+
+/// Creates a format table's schema on disk and loads the table.
+Result<std::shared_ptr<FormatTable>> CreateTable(
+ const std::shared_ptr<FileSystem>& file_system, const std::string& path,
+ const std::vector<std::string>& partition_keys,
+ const std::map<std::string, std::string>& extra_options = {}) {
+ std::map<std::string, std::string> options = {{Options::TYPE, "format-table"},
+ {Options::FILE_FORMAT, "parquet"}};
+ for (const auto& [key, value] : extra_options) {
+ options[key] = value;
+ }
+ SchemaManager schema_manager(file_system, path);
+ PAIMON_ASSIGN_OR_RAISE(
+ std::unique_ptr<TableSchema> table_schema,
+ schema_manager.CreateTable(MakeSchema(), partition_keys, /*primary_keys=*/{}, options));
+ return FormatTable::Create(file_system, path, Identifier("db", "tbl"), /*dynamic_options=*/{});
+}
+
+/// Builds one batch of the table's columns, inserts unless other row kinds are given.
+Result<std::unique_ptr<RecordBatch>> MakeBatch(
+ const std::vector<int32_t>& ids, const std::vector<std::string>& names, const std::string& dt,
+ const std::map<std::string, std::string>& partition,
+ const std::vector<RecordBatch::RowKind>& row_kinds = {}) {
+ arrow::Int32Builder id_builder;
+ arrow::StringBuilder name_builder;
+ arrow::StringBuilder dt_builder;
+ for (size_t i = 0; i < ids.size(); i++) {
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(id_builder.Append(ids[i]));
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(name_builder.Append(names[i]));
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(dt_builder.Append(dt));
+ }
+ std::shared_ptr<arrow::Array> id_array;
+ std::shared_ptr<arrow::Array> name_array;
+ std::shared_ptr<arrow::Array> dt_array;
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(id_builder.Finish(&id_array));
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(name_builder.Finish(&name_array));
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(dt_builder.Finish(&dt_array));
+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+ std::shared_ptr<arrow::StructArray> struct_array,
+ arrow::StructArray::Make({id_array, name_array, dt_array}, MakeSchema()->fields()));
+
+ auto c_array = std::make_unique<ArrowArray>();
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*struct_array, c_array.get()));
+ RecordBatchBuilder builder(c_array.get());
+ builder.SetPartition(partition);
+ if (!row_kinds.empty()) {
+ builder.SetRowKinds(row_kinds);
+ }
+ return builder.Finish();
+}
+
+/// Builds a one-row batch whose partition column is null, declared as `default_partition_name`.
+Result<std::unique_ptr<RecordBatch>> MakeBatchWithNullPartition(
+ const std::string& default_partition_name) {
+ arrow::Int32Builder id_builder;
+ arrow::StringBuilder name_builder;
+ arrow::StringBuilder dt_builder;
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(id_builder.Append(1));
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(name_builder.Append("alice"));
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(dt_builder.AppendNull());
+ std::shared_ptr<arrow::Array> id_array;
+ std::shared_ptr<arrow::Array> name_array;
+ std::shared_ptr<arrow::Array> dt_array;
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(id_builder.Finish(&id_array));
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(name_builder.Finish(&name_array));
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(dt_builder.Finish(&dt_array));
+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+ std::shared_ptr<arrow::StructArray> struct_array,
+ arrow::StructArray::Make({id_array, name_array, dt_array}, MakeSchema()->fields()));
+
+ auto c_array = std::make_unique<ArrowArray>();
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*struct_array, c_array.get()));
+ RecordBatchBuilder builder(c_array.get());
+ builder.SetPartition({{"dt", default_partition_name}});
+ return builder.Finish();
+}
+
+/// Builds one batch of `count` rows, large enough that a small split target has to cut it up.
+Result<std::unique_ptr<RecordBatch>> MakeManyRowBatch(int32_t count) {
+ std::vector<int32_t> ids;
+ std::vector<std::string> names;
+ ids.reserve(count);
+ names.reserve(count);
+ for (int32_t i = 0; i < count; i++) {
+ ids.push_back(i);
+ names.push_back("name-" + std::to_string(i));
+ }
+ return MakeBatch(ids, names, "20240101", {});
+}
+
+/// Builds one batch of `count` wide, all-different rows starting at `start_id`. A file is
+/// measured by the bytes its writer has finished with, and small repeated values sit in a
+/// dictionary it has not written yet.
+Result<std::unique_ptr<RecordBatch>> MakeWideRowBatch(int32_t count, int32_t start_id) {
+ constexpr size_t kNameLength = 1024;
+ std::vector<int32_t> ids;
+ std::vector<std::string> names;
+ ids.reserve(count);
+ names.reserve(count);
+ for (int32_t i = 0; i < count; i++) {
+ const int32_t id = start_id + i;
+ ids.push_back(id);
+ names.push_back(std::to_string(id) +
+ std::string(kNameLength, static_cast<char>('a' + (id % 26))));
+ }
+ return MakeBatch(ids, names, "20240101", {});
+}
+
+/// The path a write stages `file_path` under: a `_temporary` directory beside where the file will
+/// be published, holding a hidden name of its own. Java Paimon's `RenamingTwoPhaseOutputStream`
+/// uses the same layout.
+std::string StagedPath(const std::string& file_path) {
+ return PathUtil::JoinPath(PathUtil::GetParentDirPath(file_path),
+ "_temporary/.tmp.d9b7f0a2-0c11-4a35-9f6e-2f2f0f9e6c41");
+}
+
+/// Writes one batch and commits it, so the files become part of the table.
+Status WriteAndCommit(const std::shared_ptr<FormatTable>& table,
+ std::unique_ptr<RecordBatch>&& batch, bool overwrite = false,
+ const std::map<std::string, std::string>& static_partition = {}) {
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FormatTableWrite> write,
+ FormatTableWrite::Create(table, /*pool=*/nullptr));
+ PAIMON_RETURN_NOT_OK(write->Write(std::move(batch)));
+ PAIMON_ASSIGN_OR_RAISE(std::vector<FormatCommitMessage> messages, write->PrepareCommit());
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FormatTableCommit> commit,
+ FormatTableCommit::Create(table, overwrite, static_partition));
+ return commit->Commit(messages);
+}
+
+/// Imports a batch the reader handed out. `ASSERT_OK_AND_ASSIGN` only understands paimon's
+/// `Result`, so arrow's has to be converted before a test body can use it.
+Result<std::shared_ptr<arrow::RecordBatch>> ImportBatch(const BatchReader::ReadBatch& batch) {
+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+ std::shared_ptr<arrow::RecordBatch> record_batch,
+ arrow::ImportRecordBatch(batch.first.get(), batch.second.get()));
+ return record_batch;
+}
+
+/// Reads every row of a plan, returning the rows as `id|name|dt` strings.
+Result<std::vector<std::string>> ReadAll(const std::shared_ptr<FormatTable>& table,
+ const std::vector<std::shared_ptr<Split>>& splits,
+ const std::shared_ptr<Predicate>& predicate = nullptr,
+ bool enable_predicate_filter = false) {
+ PAIMON_ASSIGN_OR_RAISE(
+ std::unique_ptr<FormatTableRead> read,
+ FormatTableRead::TEST_Create(table, /*projection=*/std::nullopt, /*pool=*/nullptr,
+ predicate, enable_predicate_filter));
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> reader, read->CreateReader(splits));
+ std::vector<std::string> rows;
+ while (true) {
+ PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader->NextBatch());
+ if (BatchReader::IsEofBatch(batch)) {
+ break;
+ }
+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+ std::shared_ptr<arrow::RecordBatch> record_batch,
+ arrow::ImportRecordBatch(batch.first.get(), batch.second.get()));
+ // The leading field is `_VALUE_KIND`, which every `BatchReader` puts first; a format
+ // table has no row kinds of its own, so every row of it is an insert.
+ if (record_batch->schema()->field(0)->name() != SpecialFields::ValueKind().Name()) {
+ return Status::Invalid("a format table read must still carry the _VALUE_KIND field");
+ }
+ auto row_kinds = checked_pointer_cast<arrow::Int8Array>(record_batch->column(0));
+ for (int64_t i = 0; i < record_batch->num_rows(); i++) {
+ if (row_kinds->Value(i) != RowKind::Insert()->ToByteValue()) {
+ return Status::Invalid("a format table read must return inserts only");
+ }
+ }
+ auto ids = checked_pointer_cast<arrow::Int32Array>(record_batch->column(1));
+ auto names = checked_pointer_cast<arrow::StringArray>(record_batch->column(2));
+ auto dts = checked_pointer_cast<arrow::StringArray>(record_batch->column(3));
+ for (int64_t i = 0; i < record_batch->num_rows(); i++) {
+ rows.push_back(std::to_string(ids->Value(i)) + "|" + names->GetString(i) + "|" +
+ dts->GetString(i));
+ }
+ }
+ reader->Close();
+ return rows;
+}
+
+} // namespace
+
+TEST(FormatTableTest, TestParseFormat) {
+ ASSERT_OK_AND_ASSIGN(FormatTable::Format parquet, FormatTable::ParseFormat("PARQUET"));
+ ASSERT_EQ(parquet, FormatTable::Format::PARQUET);
+ ASSERT_OK_AND_ASSIGN(FormatTable::Format orc, FormatTable::ParseFormat("orc"));
+ ASSERT_EQ(orc, FormatTable::Format::ORC);
+ ASSERT_EQ(FormatTable::FormatToString(FormatTable::Format::ORC), "orc");
+
+ // Format table formats with no reader here yet answer `NotImplemented`, which is a different
+ // answer from a name that is no format at all.
+ for (const char* format : {"csv", "text", "json", "mosaic"}) {
+ Result<FormatTable::Format> unimplemented = FormatTable::ParseFormat(format);
+ ASSERT_FALSE(unimplemented.ok()) << format;
+ ASSERT_TRUE(unimplemented.status().IsNotImplemented()) << format;
+ }
+
+ Result<FormatTable::Format> unknown = FormatTable::ParseFormat("nonesuch");
+ ASSERT_FALSE(unknown.ok());
+ ASSERT_TRUE(unknown.status().IsInvalid());
+}
+
+TEST(FormatTableTest, TestCreateReadsOptions) {
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> table,
+ CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"}));
+ ASSERT_EQ(table->Location(), dir->Str());
+ ASSERT_EQ(table->GetFormat(), FormatTable::Format::PARQUET);
+ ASSERT_EQ(table->PartitionKeys(), std::vector<std::string>({"dt"}));
+ ASSERT_EQ(table->FileCompression(), "snappy");
+ ASSERT_EQ(table->PartitionDefaultName(), "__DEFAULT_PARTITION__");
+ ASSERT_EQ(table->FullName(), "db.tbl");
+}
+
+TEST(FormatTableTest, TestFileCompressionComesFromCoreOptions) {
+ // The resolution order itself is `CoreOptionsTest.TestFormatTableFileCompression`'s business;
+ // what matters here is that the table asks for it rather than resolving compression again.
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<FormatTable> table,
+ CreateTable(dir->GetFileSystem(), dir->Str(), {},
+ {{Options::FORMAT_TABLE_FILE_COMPRESSION, "lz4"}, {"compression", "zstd"}}));
+ ASSERT_EQ(table->FileCompression(), "lz4");
+
+ std::unique_ptr<UniqueTestDirectory> default_dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(default_dir);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> default_table,
+ CreateTable(default_dir->GetFileSystem(), default_dir->Str(), {}));
+ ASSERT_EQ(default_table->FileCompression(), "snappy");
+}
+
+TEST(FormatTableTest, TestFileFormatDefaultsToParquet) {
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ SchemaManager schema_manager(dir->GetFileSystem(), dir->Str());
+ ASSERT_OK_AND_ASSIGN(
+ [[maybe_unused]] std::unique_ptr<TableSchema> table_schema,
+ schema_manager.CreateTable(MakeSchema(), /*partition_keys=*/{},
+ /*primary_keys=*/{}, {{Options::TYPE, "format-table"}}));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> table,
+ FormatTable::Create(dir->GetFileSystem(), dir->Str(),
+ Identifier("db", "tbl"), /*dynamic_options=*/{}));
+ ASSERT_EQ(table->GetFormat(), FormatTable::Format::PARQUET);
+ ASSERT_EQ(table->FileCompression(), "snappy");
+}
+
+TEST(FormatTableTest, TestUnknownTableTypeIsRejected) {
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ SchemaManager schema_manager(dir->GetFileSystem(), dir->Str());
+ // Refused at creation: read as a managed table it would look for snapshots it never had.
+ Result<std::unique_ptr<TableSchema>> unknown_type = schema_manager.CreateTable(
+ MakeSchema(), /*partition_keys=*/{}, /*primary_keys=*/{}, {{Options::TYPE, "nonesuch"}});
+ ASSERT_FALSE(unknown_type.ok());
+ ASSERT_TRUE(unknown_type.status().IsInvalid());
+}
+
+TEST(FormatTableTest, TestATableTypeThisLibraryCannotOpenIsRejectedAtCreation) {
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ SchemaManager schema_manager(dir->GetFileSystem(), dir->Str());
+ // A table type paimon names but this library cannot open: refused up front, with its own
+ // status code and the type quoted.
+ Result<std::unique_ptr<TableSchema>> object_table =
+ schema_manager.CreateTable(MakeSchema(), /*partition_keys=*/{}, /*primary_keys=*/{},
+ {{Options::TYPE, "object-table"}});
+ ASSERT_FALSE(object_table.ok());
+ ASSERT_TRUE(object_table.status().IsNotImplemented()) << object_table.status().ToString();
+ ASSERT_NE(std::string::npos, object_table.status().ToString().find("object-table"));
+}
+
+TEST(FormatTableTest, TestAPartitionColumnOfAnUnsupportedTypeIsRefusedUpFront) {
+ // A partition value makes the round trip through its column type on the way to a directory
+ // name and back. `BINARY` cannot, so a table partitioned by one is refused where the table is
+ // decided rather than at the first read or write of a table that already looked created.
+ std::shared_ptr<arrow::Schema> binary_schema =
+ arrow::schema({arrow::field("id", arrow::int32()), arrow::field("name", arrow::utf8()),
+ arrow::field("bin", arrow::binary())});
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ SchemaManager schema_manager(dir->GetFileSystem(), dir->Str());
+ Result<std::unique_ptr<TableSchema>> created = schema_manager.CreateTable(
+ binary_schema, /*partition_keys=*/{"bin"}, /*primary_keys=*/{},
+ {{Options::TYPE, "format-table"}, {Options::FILE_FORMAT, "parquet"}});
+ ASSERT_NOK_WITH_MSG(created, "cannot be partitioned");
+
+ // And opening one another engine wrote fails the same way, not at its first read or write.
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<TableSchema> unchecked_schema,
+ TableSchema::Create(/*schema_id=*/0, binary_schema, /*partition_keys=*/{"bin"},
+ /*primary_keys=*/{},
+ {{Options::TYPE, "format-table"}, {Options::FILE_FORMAT, "parquet"}}));
+ std::shared_ptr<DataSchema> data_schema =
+ checked_pointer_cast<DataSchema>(std::shared_ptr<TableSchema>(std::move(unchecked_schema)));
+ Result<std::shared_ptr<FormatTable>> opened =
+ FormatTable::Create(dir->GetFileSystem(), dir->Str(), Identifier("db", "tbl"), data_schema,
+ /*location_carries_paimon_metadata=*/true, /*dynamic_options=*/{});
+ ASSERT_NOK_WITH_MSG(opened, "cannot be partitioned");
+}
+
+TEST(FormatTableTest, TestCatalogManagedPartitionsAreRejected) {
+ // The option moves partition visibility into the catalog: a directory nobody registered stops
+ // being part of the table. A scan here reads the directories instead, so honouring the option
+ // by ignoring it would return partitions the catalog never registered.
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ Result<std::shared_ptr<FormatTable>> table = CreateTable(
+ dir->GetFileSystem(), dir->Str(), {"dt"}, {{Options::METASTORE_PARTITIONED_TABLE, "true"}});
+ ASSERT_FALSE(table.ok());
+ ASSERT_TRUE(table.status().IsNotImplemented());
+
+ // Explicitly turning it off is the behaviour that is implemented, so it is accepted.
+ std::unique_ptr<UniqueTestDirectory> off_dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(off_dir);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> off_table,
+ CreateTable(off_dir->GetFileSystem(), off_dir->Str(), {"dt"},
+ {{Options::METASTORE_PARTITIONED_TABLE, "false"}}));
+
+ // And the same refusal when it arrives at the call rather than in the schema. Validating the
+ // schema's own options alone would let this one through and then drop it, leaving a caller
+ // who asked for catalog-managed partitions with a scan that read the directories anyway.
+ Result<std::shared_ptr<FormatTable>> dynamic_table =
+ FormatTable::Create(off_dir->GetFileSystem(), off_dir->Str(), Identifier("db", "tbl"),
+ off_table->LatestSchema(),
+ /*location_carries_paimon_metadata=*/true,
+ {{Options::METASTORE_PARTITIONED_TABLE, "true"}});
+ ASSERT_FALSE(dynamic_table.ok());
+ ASSERT_TRUE(dynamic_table.status().IsNotImplemented()) << dynamic_table.status().ToString();
+}
+
+TEST(FormatTableTest, TestTheGenericEntryPointsValidateOptionsGivenAtTheCall) {
+ // An option a format table refuses has to be refused wherever it comes from. Every generic
+ // entry point merges what the call gave it over what the schema stored, so each runs the same
+ // checks over the merged result; otherwise a caller could set through a context what the
+ // schema would have rejected and have it silently dropped.
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr<FormatTable> table,
+ CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"}));
+ const std::map<std::string, std::string> refused = {
+ {Options::METASTORE_PARTITIONED_TABLE, "true"}};
+
+ ScanContextBuilder scan_builder(dir->Str());
+ scan_builder.SetOptions(refused);
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<ScanContext> scan_context, scan_builder.Finish());
+ ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "metastore.partitioned-table");
+
+ ReadContextBuilder read_builder(dir->Str());
+ read_builder.SetOptions(refused);
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
+ ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "metastore.partitioned-table");
+
+ WriteContextBuilder write_builder(dir->Str(), "test-user");
+ write_builder.SetOptions(refused);
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> write_context, write_builder.Finish());
+ ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)),
+ "metastore.partitioned-table");
+
+ CommitContextBuilder commit_builder(dir->Str(), "test-user");
+ commit_builder.SetOptions(refused);
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context, commit_builder.Finish());
+ ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(commit_context)),
+ "metastore.partitioned-table");
+}
+
+TEST(FormatTableTest, TestValueOnlyPartitionCannotBeNamedAfterThisTablesMetadata) {
+ // Under the value-only layout a value becomes a directory name unchanged, so `dt` of
+ // `schema` would be written over this table's own schema, and an overwrite would delete it.
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> table,
+ CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"},
+ {{Options::FORMAT_TABLE_PARTITION_PATH_ONLY_VALUE, "true"}}));
+ ASSERT_TRUE(table->LocationCarriesPaimonMetadata());
+
+ for (const std::string& reserved : {std::string("schema"), std::string("branch")}) {
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FormatTableWrite> write,
+ FormatTableWrite::Create(table, /*pool=*/nullptr));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+ MakeBatch({1}, {"alice"}, reserved, {{"dt", reserved}}));
+ SCOPED_TRACE(reserved);
+ ASSERT_NOK_WITH_MSG(write->Write(std::move(batch)), "own metadata rather than data");
+ ASSERT_OK(write->Abort());
+ }
+
+ // An overwrite names its partition itself, so it is refused on its own account, before it
+ // deletes anything.
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FormatTableCommit> commit,
+ FormatTableCommit::Create(table, /*overwrite=*/true, {{"dt", "schema"}}));
+ ASSERT_NOK_WITH_MSG(commit->Commit({}), "own metadata rather than data");
+ // The schema is still where the table keeps it.
+ SchemaManager schema_manager(dir->GetFileSystem(), dir->Str());
+ ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<TableSchema>> latest,
+ schema_manager.Latest());
+ ASSERT_TRUE(latest.has_value());
+
+ // Any other value is ordinary data and still works.
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> ordinary,
+ MakeBatch({2}, {"bob"}, "schematic", {{"dt", "schematic"}}));
+ ASSERT_OK(WriteAndCommit(table, std::move(ordinary)));
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatTableScan> scan,
+ FormatTableScan::Create(table, /*partition_filter=*/{}, /*limit=*/std::nullopt));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan, scan->CreatePlan());
+ ASSERT_OK_AND_ASSIGN(std::vector<std::string> rows, ReadAll(table, plan->Splits()));
+ ASSERT_EQ(rows, (std::vector<std::string>{"2|bob|schematic"}));
+}
+
+TEST(FormatTableTest, TestValueOnlyPartitionCannotBeNamedByAHiddenValue) {
+ // Under the value-only layout the partition value is the whole directory name, so a value
+ // starting with `_` or `.` names a directory every scan skips: the rows would be written and
+ // never read back. The write is refused instead of losing them quietly.
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> table,
+ CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"},
+ {{Options::FORMAT_TABLE_PARTITION_PATH_ONLY_VALUE, "true"}}));
+ for (const std::string& hidden : {std::string("_2025"), std::string(".2025")}) {
+ SCOPED_TRACE(hidden);
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FormatTableWrite> write,
+ FormatTableWrite::Create(table, /*pool=*/nullptr));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+ MakeBatch({1}, {"alice"}, hidden, {{"dt", hidden}}));
+ ASSERT_NOK_WITH_MSG(write->Write(std::move(batch)), "a scan of this table would skip");
+ ASSERT_OK(write->Abort());
+ }
+
+ // The one hidden name that is table content: the directory standing for a null partition
+ // value, which this layout writes and reads like any other. What the null itself reads back
+ // as is `TestNullPartitionValue`'s business.
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> null_partition,
+ MakeBatchWithNullPartition(table->PartitionDefaultName()));
+ ASSERT_OK(WriteAndCommit(table, std::move(null_partition)));
+ ASSERT_OK_AND_ASSIGN(bool default_dir_exists, dir->GetFileSystem()->Exists(PathUtil::JoinPath(
+ dir->Str(), table->PartitionDefaultName())));
+ ASSERT_TRUE(default_dir_exists);
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatTableScan> scan,
+ FormatTableScan::Create(table, /*partition_filter=*/{}, /*limit=*/std::nullopt));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan, scan->CreatePlan());
+ ASSERT_OK_AND_ASSIGN(std::vector<std::string> rows, ReadAll(table, plan->Splits()));
+ ASSERT_EQ(rows.size(), 1u);
+
+ // The same value under the `key=value` layout is ordinary data: the key in front of it makes
+ // the directory `dt=_2025`, which is not hidden at all.
+ std::unique_ptr<UniqueTestDirectory> key_value_dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(key_value_dir);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> key_value_table,
+ CreateTable(key_value_dir->GetFileSystem(), key_value_dir->Str(), {"dt"}));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> key_value_batch,
+ MakeBatch({1}, {"alice"}, "_2025", {{"dt", "_2025"}}));
+ ASSERT_OK(WriteAndCommit(key_value_table, std::move(key_value_batch)));
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatTableScan> key_value_scan,
+ FormatTableScan::Create(key_value_table, /*partition_filter=*/{}, /*limit=*/std::nullopt));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> key_value_plan, key_value_scan->CreatePlan());
+ ASSERT_OK_AND_ASSIGN(std::vector<std::string> key_value_rows,
+ ReadAll(key_value_table, key_value_plan->Splits()));
+ ASSERT_EQ(key_value_rows, (std::vector<std::string>{"1|alice|_2025"}));
+}
+
+TEST(FormatTableTest, TestCommitMessageCannotPublishIntoThisTablesMetadata) {
+ // The same rule applies to a message that no writer here produced, since a commit checks the
+ // message it is given rather than where it came from.
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> table,
+ CreateTable(dir->GetFileSystem(), dir->Str(), {}));
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatTableCommit> commit,
+ FormatTableCommit::Create(table, /*overwrite=*/false, /*static_partition=*/{}));
+
+ FormatCommitMessage into_metadata(StagedPath(dir->Str() + "/schema/data-a-0.parquet"),
+ dir->Str() + "/schema/data-a-0.parquet",
+ std::map<std::string, std::string>{},
+ /*record_count=*/1, /*file_size=*/1);
+ ASSERT_NOK_WITH_MSG(commit->Commit({into_metadata}), "own metadata rather than data");
+}
+
+TEST(FormatTableTest, TestLocationBoundsAreCheckedFromTheRightComponent) {
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ SchemaManager schema_manager(dir->GetFileSystem(), dir->Str());
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<TableSchema> table_schema,
+ schema_manager.CreateTable(
+ MakeSchema(), /*partition_keys=*/{}, /*primary_keys=*/{},
+ {{Options::TYPE, "format-table"}, {Options::FILE_FORMAT, "parquet"}}));
+ std::shared_ptr<DataSchema> data_schema =
+ checked_pointer_cast<DataSchema>(std::shared_ptr<TableSchema>(std::move(table_schema)));
+
+ // Every path is checked against the location, and an empty one is a prefix of nothing: it
+ // would either pass every path, including those outside the table, or fail them all.
+ ASSERT_NOK_WITH_MSG(
+ FormatTable::Create(dir->GetFileSystem(), "", Identifier("db", "tbl"), data_schema,
+ /*location_carries_paimon_metadata=*/false, /*dynamic_options=*/{}),
+ "requires a location");
+
+ // The file system root is its own separator, so what is below it starts one character in.
+ // Reading it two characters in would take `/data.parquet` for `ata.parquet`, and
+ // `/.hidden.parquet` for a name that is not hidden at all.
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<FormatTable> root_table,
+ FormatTable::Create(dir->GetFileSystem(), "/", Identifier("db", "tbl"), data_schema,
+ /*location_carries_paimon_metadata=*/false, /*dynamic_options=*/{}));
+ ASSERT_EQ(root_table->Location(), "/");
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatTableRead> read,
+ FormatTableRead::TEST_Create(root_table, /*projection=*/std::nullopt, /*pool=*/nullptr,
+ /*predicate=*/nullptr,
+ /*enable_predicate_filter=*/false));
+
+ auto visible = std::make_shared<FormatDataSplit>(
+ std::vector<FormatDataSplit::FileMeta>{{"/data.parquet", 0}},
+ std::map<std::string, std::string>{});
+ ASSERT_OK(read->CreateReader(std::static_pointer_cast<Split>(visible)));
+
+ auto hidden = std::make_shared<FormatDataSplit>(
+ std::vector<FormatDataSplit::FileMeta>{{"/.data.parquet", 0}},
+ std::map<std::string, std::string>{});
+ ASSERT_NOK_WITH_MSG(read->CreateReader(std::static_pointer_cast<Split>(hidden)), "would skip");
+
+ // A location written with a trailing separator names the same directory as one without, and
+ // bounds the same paths.
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<FormatTable> trailing,
+ FormatTable::Create(dir->GetFileSystem(), dir->Str() + "/", Identifier("db", "tbl"),
+ data_schema,
+ /*location_carries_paimon_metadata=*/false, /*dynamic_options=*/{}));
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatTableRead> trailing_read,
+ FormatTableRead::TEST_Create(trailing, /*projection=*/std::nullopt, /*pool=*/nullptr,
+ /*predicate=*/nullptr,
+ /*enable_predicate_filter=*/false));
+ auto inside = std::make_shared<FormatDataSplit>(
+ std::vector<FormatDataSplit::FileMeta>{{dir->Str() + "/data.parquet", 0}},
+ std::map<std::string, std::string>{});
+ ASSERT_OK(trailing_read->CreateReader(std::static_pointer_cast<Split>(inside)));
+ auto outside = std::make_shared<FormatDataSplit>(
+ std::vector<FormatDataSplit::FileMeta>{{dir->Str() + "-sibling/data.parquet", 0}},
+ std::map<std::string, std::string>{});
+ ASSERT_NOK_WITH_MSG(trailing_read->CreateReader(std::static_pointer_cast<Split>(outside)),
+ "not under the table location");
+}
+
+TEST(FormatTableTest, TestATrailingSeparatorStillNamesTheTableLocation) {
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr<FormatTable> created,
+ CreateTable(dir->GetFileSystem(), dir->Str(), {}));
+ SchemaManager schema_manager(dir->GetFileSystem(), dir->Str());
+ ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<TableSchema>> latest,
+ schema_manager.Latest());
+ ASSERT_TRUE(latest.has_value());
+ std::shared_ptr<DataSchema> data_schema = checked_pointer_cast<DataSchema>(latest.value());
+
+ // The same table with a trailing separator. Whether `schema` below the location is metadata
+ // is decided by comparing the two, so both spellings must compare equal; otherwise an
+ // overwrite at the root would list the schema as data and delete it.
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<FormatTable> table,
+ FormatTable::Create(dir->GetFileSystem(), dir->Str() + "/", Identifier("db", "tbl"),
+ data_schema,
+ /*location_carries_paimon_metadata=*/true, /*dynamic_options=*/{}));
+
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+ MakeBatch({1}, {"alice"}, "20240101", {}));
+ ASSERT_OK(WriteAndCommit(table, std::move(batch), /*overwrite=*/true));
+
+ // The schema is still where the table keeps it, and the rows are readable.
+ ASSERT_OK_AND_ASSIGN(std::optional<std::shared_ptr<TableSchema>> after,
+ schema_manager.Latest());
+ ASSERT_TRUE(after.has_value());
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatTableScan> scan,
+ FormatTableScan::Create(table, /*partition_filter=*/{}, /*limit=*/std::nullopt));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan, scan->CreatePlan());
+ ASSERT_OK_AND_ASSIGN(std::vector<std::string> rows, ReadAll(table, plan->Splits()));
+ ASSERT_EQ(rows, (std::vector<std::string>{"1|alice|20240101"}));
+}
+
+TEST(FormatTableTest, TestExternalLocationHasNoReservedDirectories) {
+ // A format table served by a catalog that keeps schemas elsewhere has nothing but data below
+ // its location. A value-only partition whose value happens to be `schema` is such data, and
+ // skipping it would drop rows without a word.
+ std::unique_ptr<UniqueTestDirectory> schema_dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(schema_dir);
+ std::map<std::string, std::string> options = {
+ {Options::TYPE, "format-table"},
+ {Options::FILE_FORMAT, "parquet"},
+ {Options::FORMAT_TABLE_PARTITION_PATH_ONLY_VALUE, "true"}};
+ SchemaManager schema_manager(schema_dir->GetFileSystem(), schema_dir->Str());
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<TableSchema> table_schema,
+ schema_manager.CreateTable(MakeSchema(), /*partition_keys=*/{"dt"},
+ /*primary_keys=*/{}, options));
+
+ // The data lives somewhere else entirely, the way an external table's does.
+ std::unique_ptr<UniqueTestDirectory> data_dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(data_dir);
+ std::shared_ptr<DataSchema> data_schema =
+ checked_pointer_cast<DataSchema>(std::shared_ptr<TableSchema>(std::move(table_schema)));
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<FormatTable> external,
+ FormatTable::Create(data_dir->GetFileSystem(), data_dir->Str(), Identifier("db", "tbl"),
+ data_schema,
+ /*location_carries_paimon_metadata=*/false, /*dynamic_options=*/{}));
+ ASSERT_FALSE(external->LocationCarriesPaimonMetadata());
+
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+ MakeBatch({1}, {"alice"}, "schema", {{"dt", "schema"}}));
+ ASSERT_OK(WriteAndCommit(external, std::move(batch)));
+
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatTableScan> scan,
+ FormatTableScan::Create(external, /*partition_filter=*/{}, /*limit=*/std::nullopt));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan, scan->CreatePlan());
+ ASSERT_OK_AND_ASSIGN(std::vector<std::string> rows, ReadAll(external, plan->Splits()));
+ ASSERT_EQ(rows, (std::vector<std::string>{"1|alice|schema"}));
+}
+
+TEST(FormatTableTest, TestFileSuffixIncludesCompressionWhenAsked) {
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ // A format that records its own compression keeps a plain name unless the option asks for it,
+ // and then the compression goes in front of the format: `data-<uuid>-0.snappy.parquet`.
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> table,
+ CreateTable(dir->GetFileSystem(), dir->Str(), {},
+ {{Options::FILE_FORMAT, "parquet"},
+ {Options::FILE_SUFFIX_INCLUDE_COMPRESSION, "true"}}));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FormatTableWrite> write,
+ FormatTableWrite::Create(table, /*pool=*/nullptr));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+ MakeBatch({1}, {"alice"}, "20240101", {}));
+ ASSERT_OK(write->Write(std::move(batch)));
+ ASSERT_OK_AND_ASSIGN(std::vector<FormatCommitMessage> messages, write->PrepareCommit());
+ ASSERT_EQ(messages.size(), 1u);
+ ASSERT_TRUE(StringUtils::EndsWith(messages[0].file_path, ".snappy.parquet"))
+ << messages[0].file_path;
+ ASSERT_OK(write->Abort());
+
+ // A compression hadoop does not name goes into the file name as the option spelled it.
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> uncompressed,
+ CreateTable(dir->GetFileSystem(), dir->Str() + "/other", {},
+ {{Options::FILE_FORMAT, "parquet"},
+ {Options::FILE_COMPRESSION, "uncompressed"},
+ {Options::FILE_SUFFIX_INCLUDE_COMPRESSION, "true"}}));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FormatTableWrite> other_write,
+ FormatTableWrite::Create(uncompressed, /*pool=*/nullptr));
+ ASSERT_OK_AND_ASSIGN(batch, MakeBatch({1}, {"alice"}, "20240101", {}));
+ ASSERT_OK(other_write->Write(std::move(batch)));
+ ASSERT_OK_AND_ASSIGN(messages, other_write->PrepareCommit());
+ ASSERT_EQ(messages.size(), 1u);
+ ASSERT_TRUE(StringUtils::EndsWith(messages[0].file_path, ".uncompressed.parquet"))
+ << messages[0].file_path;
+ ASSERT_OK(other_write->Abort());
+}
+
+TEST(FormatTableTest, TestValueOnlyLayoutNeedsPartitionKeys) {
+ // The layout names a directory by its partition value alone, so a table with no partition
+ // keys asks for a layout that has nothing to lay out.
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ ASSERT_NOK_WITH_MSG(CreateTable(dir->GetFileSystem(), dir->Str(), {},
+ {{Options::FORMAT_TABLE_PARTITION_PATH_ONLY_VALUE, "true"}}),
+ "on a table with no partition keys");
+}
+
+TEST(FormatTableTest, TestCreateRejectsManagedTable) {
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ SchemaManager schema_manager(dir->GetFileSystem(), dir->Str());
+ ASSERT_OK_AND_ASSIGN(
+ [[maybe_unused]] std::unique_ptr<TableSchema> table_schema,
+ schema_manager.CreateTable(MakeSchema(), /*partition_keys=*/{}, /*primary_keys=*/{},
+ {{Options::FILE_FORMAT, "parquet"}}));
+ Result<std::shared_ptr<FormatTable>> table = FormatTable::Create(
+ dir->GetFileSystem(), dir->Str(), Identifier("db", "tbl"), /*dynamic_options=*/{});
+ ASSERT_FALSE(table.ok());
+ ASSERT_TRUE(table.status().IsInvalid());
+}
+
+TEST(FormatTableTest, TestValueOnlyPartitionLayout) {
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> table,
+ CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"},
+ {{Options::FORMAT_TABLE_PARTITION_PATH_ONLY_VALUE, "true"}}));
+ ASSERT_TRUE(table->PartitionOnlyValueInPath());
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+ MakeBatch({1, 2}, {"alice", "bob"}, "20240101", {{"dt", "20240101"}}));
+ ASSERT_OK(WriteAndCommit(table, std::move(batch)));
+
+ // The directory is the bare value, with no field name in it.
+ ASSERT_OK_AND_ASSIGN(bool value_only_dir_exists,
+ dir->GetFileSystem()->Exists(PathUtil::JoinPath(dir->Str(), "20240101")));
+ ASSERT_TRUE(value_only_dir_exists);
+ ASSERT_OK_AND_ASSIGN(bool key_value_dir_exists, dir->GetFileSystem()->Exists(PathUtil::JoinPath(
+ dir->Str(), "dt=20240101")));
+ ASSERT_FALSE(key_value_dir_exists);
+
+ // And the scan reads back the layout the write produced.
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatTableScan> scan,
+ FormatTableScan::Create(table, /*partition_filter=*/{}, /*limit=*/std::nullopt));
+ ASSERT_OK_AND_ASSIGN(PartitionList partitions, scan->ListPartitions());
+ ASSERT_EQ(partitions.size(), 1);
+ ASSERT_EQ(partitions[0].at("dt"), "20240101");
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan, scan->CreatePlan());
+ ASSERT_OK_AND_ASSIGN(std::vector<std::string> rows, ReadAll(table, plan->Splits()));
+ ASSERT_EQ(rows.size(), 2);
+ ASSERT_EQ(rows[0], "1|alice|20240101");
+}
+
+TEST(FormatTableTest, TestScanFindsDataFilesInSubdirectories) {
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> table,
+ CreateTable(dir->GetFileSystem(), dir->Str(), {}));
+
+ // A real data file, written by this table and then moved: `data-file.path-directory` puts
+ // data files a level down, and an engine writing the directory may do the same. The partition
+ // columns are not in the file, so nesting changes nothing about what one holds.
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+ MakeBatch({1, 2}, {"alice", "bob"}, "20240101", {}));
+ ASSERT_OK(WriteAndCommit(table, std::move(batch)));
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatTableScan> written,
+ FormatTableScan::Create(table, /*partition_filter=*/{}, /*limit=*/std::nullopt));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> written_plan, written->CreatePlan());
+ ASSERT_EQ(written_plan->Splits().size(), 1u);
+ auto written_split = std::dynamic_pointer_cast<FormatDataSplit>(written_plan->Splits()[0]);
+ ASSERT_NE(written_split, nullptr);
+ ASSERT_EQ(written_split->files.size(), 1u);
+ const std::string written_path = written_split->files[0].file_path;
+
+ std::string nested = PathUtil::JoinPath(dir->Str(), "bucket-0");
+ ASSERT_OK(dir->GetFileSystem()->Mkdirs(nested));
+ const std::string nested_path = PathUtil::JoinPath(nested, PathUtil::GetName(written_path));
+ ASSERT_OK(dir->GetFileSystem()->Rename(written_path, nested_path));
+
+ // A staging tree at the same level stays invisible, whatever its files are called, even when
+ // the file in it is a perfectly readable copy of the one above.
+ std::string staging = PathUtil::JoinPath(dir->Str(), "_temporary");
+ ASSERT_OK(dir->GetFileSystem()->Mkdirs(staging));
+ std::string content;
+ ASSERT_OK(dir->GetFileSystem()->ReadFile(nested_path, &content));
+ ASSERT_OK(dir->GetFileSystem()->WriteFile(
+ PathUtil::JoinPath(staging, PathUtil::GetName(written_path)), content,
+ /*overwrite=*/true));
+
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatTableScan> scan,
+ FormatTableScan::Create(table, /*partition_filter=*/{}, /*limit=*/std::nullopt));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan, scan->CreatePlan());
+ ASSERT_OK_AND_ASSIGN(std::vector<std::string> rows, ReadAll(table, plan->Splits()));
+ ASSERT_EQ(rows.size(), 2u);
+ ASSERT_EQ(rows[0], "1|alice|20240101");
+ ASSERT_EQ(rows[1], "2|bob|20240101");
+}
+
+TEST(FormatTableTest, TestTargetFileRowNumRollsFiles) {
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<FormatTable> table,
+ CreateTable(dir->GetFileSystem(), dir->Str(), {}, {{Options::TARGET_FILE_ROW_NUM, "1"}}));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FormatTableWrite> write,
+ FormatTableWrite::Create(table, /*pool=*/nullptr));
+ for (int32_t i = 0; i < 3; i++) {
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+ MakeBatch({i}, {"name"}, "20240101", {}));
+ ASSERT_OK(write->Write(std::move(batch)));
+ }
+ ASSERT_OK_AND_ASSIGN(std::vector<FormatCommitMessage> messages, write->PrepareCommit());
+ // Rolling is checked between batches, so each one-row batch closes its own file.
+ ASSERT_EQ(messages.size(), 3);
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatTableCommit> commit,
+ FormatTableCommit::Create(table, /*overwrite=*/false, /*static_partition=*/{}));
+ ASSERT_OK(commit->Commit(messages));
+}
+
+TEST(FormatTableTest, TestTargetFileSizeRollsFiles) {
+ constexpr int32_t kBatches = 2;
+ constexpr int32_t kRowsPerBatch = 2000;
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ // Rolling is checked between batches, so each batch is weighed against the target and a file
+ // closes once it is past it. Without this a write would put a whole partition in one file
+ // however large it grew.
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<FormatTable> table,
+ CreateTable(dir->GetFileSystem(), dir->Str(), {}, {{Options::TARGET_FILE_SIZE, "1 kb"}}));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FormatTableWrite> write,
+ FormatTableWrite::Create(table, /*pool=*/nullptr));
+ // Megabytes of values, each of them different: a file is weighed by what its writer has
+ // finished with, and a writer holds on to what it can still encode more cheaply later.
+ for (int32_t batch = 0; batch < kBatches; batch++) {
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> rows,
+ MakeWideRowBatch(kRowsPerBatch, batch * kRowsPerBatch));
+ ASSERT_OK(write->Write(std::move(rows)));
+ }
+ ASSERT_OK_AND_ASSIGN(std::vector<FormatCommitMessage> messages, write->PrepareCommit());
+ ASSERT_GT(messages.size(), 1u);
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatTableCommit> commit,
+ FormatTableCommit::Create(table, /*overwrite=*/false, /*static_partition=*/{}));
+ ASSERT_OK(commit->Commit(messages));
+
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatTableScan> scan,
+ FormatTableScan::Create(table, /*partition_filter=*/{}, /*limit=*/std::nullopt));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan, scan->CreatePlan());
+ ASSERT_OK_AND_ASSIGN(std::vector<std::string> read_rows, ReadAll(table, plan->Splits()));
+ ASSERT_EQ(read_rows.size(), static_cast<size_t>(kBatches) * static_cast<size_t>(kRowsPerBatch));
+}
+
+TEST(FormatTableTest, TestANonPositiveLimitPlansNothing) {
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> table,
+ CreateTable(dir->GetFileSystem(), dir->Str(), {}));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+ MakeBatch({1, 2}, {"alice", "bob"}, "20240101", {}));
+ ASSERT_OK(WriteAndCommit(table, std::move(batch)));
+
+ // A limit of zero asks for no rows, so there is nothing to read and no split to hand out. A
+ // positive limit cannot drop anything, since a format table records no row counts.
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FormatTableScan> none,
+ FormatTableScan::Create(table, /*partition_filter=*/{}, /*limit=*/0));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> empty_plan, none->CreatePlan());
+ ASSERT_TRUE(empty_plan->Splits().empty());
+
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FormatTableScan> some,
+ FormatTableScan::Create(table, /*partition_filter=*/{}, /*limit=*/1));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan, some->CreatePlan());
+ ASSERT_FALSE(plan->Splits().empty());
+}
+
+TEST(FormatTableTest, TestSplitsAreBoundedByTheSplitTargetSize) {
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ // One file per split: every file costs at least the open-file cost, which fills a split on its
+ // own. Without packing the whole partition would be a single split however many files it has.
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> table,
+ CreateTable(dir->GetFileSystem(), dir->Str(), {},
+ {{Options::TARGET_FILE_ROW_NUM, "1"},
+ {Options::SOURCE_SPLIT_TARGET_SIZE, "1 kb"},
+ {Options::SOURCE_SPLIT_OPEN_FILE_COST, "1 kb"}}));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FormatTableWrite> write,
+ FormatTableWrite::Create(table, /*pool=*/nullptr));
+ for (int32_t i = 0; i < 3; i++) {
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+ MakeBatch({i}, {"name"}, "20240101", {}));
+ ASSERT_OK(write->Write(std::move(batch)));
+ }
+ ASSERT_OK_AND_ASSIGN(std::vector<FormatCommitMessage> messages, write->PrepareCommit());
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatTableCommit> commit,
+ FormatTableCommit::Create(table, /*overwrite=*/false, /*static_partition=*/{}));
+ ASSERT_OK(commit->Commit(messages));
+
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatTableScan> scan,
+ FormatTableScan::Create(table, /*partition_filter=*/{}, /*limit=*/std::nullopt));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan, scan->CreatePlan());
+ ASSERT_EQ(plan->Splits().size(), 3);
+ // Every row is still read exactly once, whichever split it landed in.
+ ASSERT_OK_AND_ASSIGN(std::vector<std::string> rows, ReadAll(table, plan->Splits()));
+ ASSERT_EQ(rows.size(), 3);
+}
+
+TEST(FormatTableTest, TestAFileLargerThanTheSplitTargetIsStillOneSplit) {
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ // One file, far larger than a split.
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> table,
+ CreateTable(dir->GetFileSystem(), dir->Str(), {},
+ {{Options::SOURCE_SPLIT_TARGET_SIZE, "1 kb"},
+ {Options::SOURCE_SPLIT_OPEN_FILE_COST, "1 kb"}}));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch, MakeManyRowBatch(500));
+ ASSERT_OK(WriteAndCommit(table, std::move(batch)));
+
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatTableScan> scan,
+ FormatTableScan::Create(table, /*partition_filter=*/{}, /*limit=*/std::nullopt));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan, scan->CreatePlan());
+ // A split holds whole files, because parquet and orc record their own row group and stripe
+ // boundaries, so a file past the target size is one split rather than several.
+ ASSERT_EQ(plan->Splits().size(), 1u);
+
+ // And every row of it is read exactly once.
+ ASSERT_OK_AND_ASSIGN(std::vector<std::string> rows, ReadAll(table, plan->Splits()));
+ ASSERT_EQ(rows.size(), 500u);
+ std::vector<std::string> expected;
+ expected.reserve(500);
+ for (int32_t i = 0; i < 500; i++) {
+ expected.push_back(std::to_string(i) + "|name-" + std::to_string(i) + "|20240101");
+ }
+ std::sort(rows.begin(), rows.end());
+ std::sort(expected.begin(), expected.end());
+ ASSERT_EQ(rows, expected);
+}
+
+TEST(FormatTableTest, TestBlankPartitionValueLandsInTheDefaultPartition) {
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> table,
+ CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"}));
+ // A null, an empty string and a whitespace-only string alike stand for the default partition
+ // name. Treating only null that way would put the other two in directories of their own, and
+ // an empty one has no legal directory name at all.
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<RecordBatch> blank,
+ MakeBatch({1, 2}, {"alice", "bob"}, " ", {{"dt", table->PartitionDefaultName()}}));
+ ASSERT_OK(WriteAndCommit(table, std::move(blank)));
+
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatTableScan> scan,
+ FormatTableScan::Create(table, /*partition_filter=*/{}, /*limit=*/std::nullopt));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan, scan->CreatePlan());
+ ASSERT_OK_AND_ASSIGN(PartitionList partitions, scan->ListPartitions());
+ ASSERT_EQ(partitions.size(), 1u);
+ ASSERT_EQ(partitions[0].at("dt"), table->PartitionDefaultName());
+ // The value reads back as null, as it does for a null partition.
+ ASSERT_OK_AND_ASSIGN(std::vector<std::string> rows, ReadAll(table, plan->Splits()));
+ ASSERT_EQ(rows.size(), 2u);
+}
+
+TEST(FormatTableTest, TestAFileThatCannotBeClosedEndsTheWrite) {
+ // A store can refuse `Flush()`, `GetPos()` or `Close()` on the stream a file is written
+ // through, and each lands at a different point: opening the file, adding to it, or closing it.
+ // Wherever it lands, the write answers with the failure and hands out nothing to publish,
+ // rather than reaching through a writer that is no longer there.
+ for (const FailingStreamCall failing :
+ {FailingStreamCall::kGetPos, FailingStreamCall::kFlush, FailingStreamCall::kClose}) {
+ SCOPED_TRACE(static_cast<int32_t>(failing));
+ // Rolling on every row, so a refusal in the closing path lands inside `Write()`.
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr<FormatTable> created,
+ CreateTable(dir->GetFileSystem(), dir->Str(), {},
+ {{Options::TARGET_FILE_ROW_NUM, "1"}}));
+ auto failing_fs = std::make_shared<FailingWriteFileSystem>(dir->GetFileSystem(), failing);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> table,
+ FormatTable::Create(failing_fs, dir->Str(), Identifier("db", "tbl"),
+ /*dynamic_options=*/{}));
+
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FormatTableWrite> write,
+ FormatTableWrite::Create(table, /*pool=*/nullptr));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first,
+ MakeBatch({1}, {"alice"}, "20240101", {}));
+ ASSERT_NOK(write->Write(std::move(first)));
+
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second,
+ MakeBatch({2}, {"bob"}, "20240101", {}));
+ ASSERT_NOK(write->Write(std::move(second)));
+ // A refusal while closing ends the write, since the rows of a file that cannot be closed
+ // can never be published. One while the file was being opened staged nothing at all, so
+ // there is nothing to refuse and nothing to hand out either.
+ Result<std::vector<FormatCommitMessage>> prepared = write->PrepareCommit();
+ if (prepared.ok()) {
+ ASSERT_TRUE(prepared.value().empty());
+ }
+ ASSERT_OK(write->Abort());
+
+ // Whatever it gave up on is gone, so a later scan does not see a partial file.
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> plain_table,
+ FormatTable::Create(dir->GetFileSystem(), dir->Str(),
+ Identifier("db", "tbl"), /*dynamic_options=*/{}));
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatTableScan> scan,
+ FormatTableScan::Create(plain_table, /*partition_filter=*/{}, /*limit=*/std::nullopt));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan, scan->CreatePlan());
+ ASSERT_TRUE(plan->Splits().empty());
+ }
+
+ // `Close()` is reached only once the writer has been finished and dropped, so a refusal there
+ // is the case worth pinning down: the write is over, and both entry points say so rather than
+ // publishing what is left or reaching through the writer that is gone.
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr<FormatTable> created,
+ CreateTable(dir->GetFileSystem(), dir->Str(), {}));
+ auto failing_fs =
+ std::make_shared<FailingWriteFileSystem>(dir->GetFileSystem(), FailingStreamCall::kClose);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> table,
+ FormatTable::Create(failing_fs, dir->Str(), Identifier("db", "tbl"),
+ /*dynamic_options=*/{}));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FormatTableWrite> write,
+ FormatTableWrite::Create(table, /*pool=*/nullptr));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+ MakeBatch({1}, {"alice"}, "20240101", {}));
+ ASSERT_OK(write->Write(std::move(batch)));
+ ASSERT_NOK(write->PrepareCommit());
+ ASSERT_NOK(write->PrepareCommit());
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> after,
+ MakeBatch({2}, {"bob"}, "20240101", {}));
+ ASSERT_NOK(write->Write(std::move(after)));
+ ASSERT_OK(write->Abort());
+}
+
+TEST(FormatTableTest, TestAFileWhoseFooterCannotBeWrittenIsGivenUpOnOnce) {
+ // A writer releases what it writes through before it writes the file's footer, so a store that
+ // refuses that last write leaves nothing to finish a second time. The cleanup after the failure
+ // must not ask it to - that would reach through what the failed finish already freed - and the
+ // caller must still be told what went wrong. orc frees its batch there, so both are walked.
+ for (const char* file_format : {"parquet", "orc"}) {
+ SCOPED_TRACE(file_format);
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr<FormatTable> created,
+ CreateTable(dir->GetFileSystem(), dir->Str(), {},
+ {{Options::FILE_FORMAT, file_format}}));
+ auto failing_fs = std::make_shared<ArmedWriteFailureFileSystem>(dir->GetFileSystem());
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> table,
+ FormatTable::Create(failing_fs, dir->Str(), Identifier("db", "tbl"),
+ /*dynamic_options=*/{}));
+
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FormatTableWrite> write,
+ FormatTableWrite::Create(table, /*pool=*/nullptr));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+ MakeBatch({1}, {"alice"}, "20240101", {}));
+ ASSERT_OK(write->Write(std::move(batch)));
+
+ // Everything the writer buffered went through; only what finishing it writes is refused.
+ failing_fs->FailFurtherWrites();
+ ASSERT_NOK_WITH_MSG(write->PrepareCommit(), "injected write failure");
+
+ // The write is over: it takes no more rows and has nothing whole to publish.
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> after,
+ MakeBatch({2}, {"bob"}, "20240101", {}));
+ ASSERT_NOK(write->Write(std::move(after)));
+ ASSERT_NOK(write->PrepareCommit());
+ ASSERT_OK(write->Abort());
+
+ // What it gave up on is gone, so a later scan does not find a file with no footer.
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> plain_table,
+ FormatTable::Create(dir->GetFileSystem(), dir->Str(),
+ Identifier("db", "tbl"), /*dynamic_options=*/{}));
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatTableScan> scan,
+ FormatTableScan::Create(plain_table, /*partition_filter=*/{}, /*limit=*/std::nullopt));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan, scan->CreatePlan());
+ ASSERT_TRUE(plan->Splits().empty());
+ }
+}
+
+TEST(FormatTableTest, TestAFailedTargetSizeCheckEndsTheWrite) {
+ // The other point inside `Write()` that reaches the file: once a batch is in, the writer is
+ // asked whether the file has grown to its target, and answers by measuring what it has written.
+ // A refusal there leaves the file part way through just as a refused write does, so the write
+ // ends rather than carrying on with a file it can no longer measure.
+ //
+ // The refusal is armed only after a batch is in. Refusing from the start would stop the write
+ // in `OpenFile()` instead - see `FailingWriteFileSystem` - which stages nothing and so is not
+ // terminal; `TestAFileThatCannotBeClosedEndsTheWrite` covers that case.
+ for (const char* file_format : {"parquet", "orc"}) {
+ SCOPED_TRACE(file_format);
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ // The row target is left at its default, so the size check is what runs.
+ ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr<FormatTable> created,
+ CreateTable(dir->GetFileSystem(), dir->Str(), {},
+ {{Options::FILE_FORMAT, file_format}}));
+ auto failing_fs = std::make_shared<FailingWriteFileSystem>(
+ dir->GetFileSystem(), FailingStreamCall::kGetPos, /*armed=*/false);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> table,
+ FormatTable::Create(failing_fs, dir->Str(), Identifier("db", "tbl"),
+ /*dynamic_options=*/{}));
+
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FormatTableWrite> write,
+ FormatTableWrite::Create(table, /*pool=*/nullptr));
+ // Opens the file and passes the size check, so the next one is what the refusal meets.
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> opening,
+ MakeBatch({1}, {"alice"}, "20240101", {}));
+ ASSERT_OK(write->Write(std::move(opening)));
+ failing_fs->Arm();
+
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+ MakeBatch({2}, {"bob"}, "20240101", {}));
+ ASSERT_NOK_WITH_MSG(write->Write(std::move(batch)), "injected get position failure");
+
+ // Terminal: the file is part way through, so neither entry point may carry on with it.
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> after,
+ MakeBatch({3}, {"carol"}, "20240101", {}));
+ ASSERT_NOK(write->Write(std::move(after)));
+ ASSERT_NOK(write->PrepareCommit());
+ ASSERT_OK(write->Abort());
+
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> plain_table,
+ FormatTable::Create(dir->GetFileSystem(), dir->Str(),
+ Identifier("db", "tbl"), /*dynamic_options=*/{}));
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatTableScan> scan,
+ FormatTableScan::Create(plain_table, /*partition_filter=*/{}, /*limit=*/std::nullopt));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan, scan->CreatePlan());
+ ASSERT_TRUE(plan->Splits().empty());
+ }
+}
+
+TEST(FormatTableTest, TestAFailedOpenClosesTheFileBeforeDeletingIt) {
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ // A compression no codec answers to, so opening the file succeeds and building the writer
+ // over it fails: the one path where a temp file exists and nothing owns it yet.
+ ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr<FormatTable> created,
+ CreateTable(dir->GetFileSystem(), dir->Str(), {},
+ {{Options::FILE_COMPRESSION, "nonesuch"}}));
+ auto recording = std::make_shared<CallOrderFileSystem>(dir->GetFileSystem());
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> table,
+ FormatTable::Create(recording, dir->Str(), Identifier("db", "tbl"),
+ /*dynamic_options=*/{}));
+
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FormatTableWrite> write,
+ FormatTableWrite::Create(table, /*pool=*/nullptr));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+ MakeBatch({1}, {"alice"}, "20240101", {}));
+ Status status = write->Write(std::move(batch));
+ ASSERT_FALSE(status.ok()) << "a writer was built over a compression nothing implements";
+
+ // Created, closed, and only then deleted. Deleting first would leave the stream to flush
+ // afterwards, and on a store that writes from its destructor the file would come back under
+ // a hidden name no scan reads and no abort knows about.
+ const std::vector<std::string>& calls = recording->Calls();
+ ASSERT_EQ(calls.size(), 3u);
+ ASSERT_TRUE(StringUtils::StartsWith(calls[0], "create ")) << calls[0];
+ ASSERT_TRUE(StringUtils::StartsWith(calls[1], "close ")) << calls[1];
+ ASSERT_TRUE(StringUtils::StartsWith(calls[2], "delete ")) << calls[2];
+ // All three name the one temp file, which is hidden so that no scan can reach it.
+ const std::string temp_path = calls[0].substr(std::string("create ").size());
+ ASSERT_EQ(calls[1], "close " + temp_path);
+ ASSERT_EQ(calls[2], "delete " + temp_path);
+ ASSERT_TRUE(StringUtils::StartsWith(PathUtil::GetName(temp_path), ".")) << temp_path;
+
+ // Nothing is left behind for a later scan to pick up.
+ ASSERT_OK_AND_ASSIGN(bool exists, dir->GetFileSystem()->Exists(temp_path));
+ ASSERT_FALSE(exists);
+}
+
+TEST(FormatTableTest, TestCreateRejectsSchemasThatCouldNeverBeUsed) {
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ // Every one of these can be persisted and loaded, and would otherwise only fail when a reader
+ // or writer is built, long after the table looked created.
+ ASSERT_NOK_WITH_MSG(CreateTable(dir->GetFileSystem(), dir->Str() + "/a", {},
+ {{Options::FILE_FORMAT, "nonesuch"}}),
+ "unsupported file format");
+ ASSERT_NOK_WITH_MSG(CreateTable(dir->GetFileSystem(), dir->Str() + "/b", {},
+ {{Options::TARGET_FILE_ROW_NUM, "0"}}),
+ "should be at least 1");
+ // A format table format with no reader here is refused by name, rather than failing later
+ // with a missing-format-factory error.
+ ASSERT_NOK_WITH_MSG(
+ CreateTable(dir->GetFileSystem(), dir->Str() + "/c", {}, {{Options::FILE_FORMAT, "csv"}}),
+ "not supported by paimon-cpp yet");
+ // Every column a partition column leaves the data files with nothing in them.
+ ASSERT_NOK_WITH_MSG(
+ CreateTable(dir->GetFileSystem(), dir->Str() + "/d", {"id", "name", "dt"}, {}),
+ "every one of its columns");
+}
+
+TEST(FormatTableTest, TestPrimaryKeysAreRejectedWhenTheTableIsCreated) {
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ SchemaManager schema_manager(dir->GetFileSystem(), dir->Str());
+ // A format table with primary keys could never be opened, so it is never written either.
+ Result<std::unique_ptr<TableSchema>> table_schema = schema_manager.CreateTable(
+ MakeSchema(), /*partition_keys=*/{}, /*primary_keys=*/{"id"},
+ {{Options::TYPE, "format-table"}, {Options::FILE_FORMAT, "parquet"}});
+ ASSERT_FALSE(table_schema.ok());
+ ASSERT_TRUE(table_schema.status().IsInvalid());
+}
+
+TEST(FormatTableTest, TestWriteReadUnpartitioned) {
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> table,
+ CreateTable(dir->GetFileSystem(), dir->Str(), {}));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+ MakeBatch({1, 2}, {"alice", "bob"}, "20240101", {}));
+ ASSERT_OK(WriteAndCommit(table, std::move(batch)));
+
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatTableScan> scan,
+ FormatTableScan::Create(table, /*partition_filter=*/{}, /*limit=*/std::nullopt));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan, scan->CreatePlan());
+ ASSERT_EQ(plan->Splits().size(), 1);
+
+ ASSERT_OK_AND_ASSIGN(std::vector<std::string> rows, ReadAll(table, plan->Splits()));
+ ASSERT_EQ(rows.size(), 2u);
+ ASSERT_EQ(rows[0], "1|alice|20240101");
+ ASSERT_EQ(rows[1], "2|bob|20240101");
+}
+
+TEST(FormatTableTest, TestBatchesOfOnePartitionShareOneFile) {
+ // The directory a partition writes into is derived once and kept, so a second batch for the
+ // same partition finds the file the first one opened instead of starting another. A partition
+ // that has not been seen before still gets a file of its own.
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> table,
+ CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"}));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FormatTableWrite> write,
+ FormatTableWrite::Create(table, /*pool=*/nullptr));
+ for (const auto& [id, name, dt] : std::vector<std::tuple<int32_t, std::string, std::string>>{
+ {1, "alice", "20240101"}, {2, "bob", "20240101"}, {3, "carol", "20240102"}}) {
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+ MakeBatch({id}, {name}, dt, {{"dt", dt}}));
+ ASSERT_OK(write->Write(std::move(batch)));
+ }
+
+ ASSERT_OK_AND_ASSIGN(std::vector<FormatCommitMessage> messages, write->PrepareCommit());
+ // One file per partition, not one per batch.
+ ASSERT_EQ(messages.size(), 2u);
+ std::sort(messages.begin(), messages.end(),
+ [](const FormatCommitMessage& left, const FormatCommitMessage& right) {
+ return left.file_path < right.file_path;
+ });
+ ASSERT_EQ(messages[0].partition, (std::map<std::string, std::string>{{"dt", "20240101"}}));
+ ASSERT_EQ(messages[0].record_count, 2);
+ ASSERT_EQ(messages[1].partition, (std::map<std::string, std::string>{{"dt", "20240102"}}));
+ ASSERT_EQ(messages[1].record_count, 1);
+
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatTableCommit> commit,
+ FormatTableCommit::Create(table, /*overwrite=*/false, /*static_partition=*/{}));
+ ASSERT_OK(commit->Commit(messages));
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatTableScan> scan,
+ FormatTableScan::Create(table, /*partition_filter=*/{}, /*limit=*/std::nullopt));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan, scan->CreatePlan());
+ ASSERT_OK_AND_ASSIGN(std::vector<std::string> rows, ReadAll(table, plan->Splits()));
+ std::sort(rows.begin(), rows.end());
+ ASSERT_EQ(rows,
+ (std::vector<std::string>{"1|alice|20240101", "2|bob|20240101", "3|carol|20240102"}));
+}
+
+TEST(FormatTableTest, TestWriteReadPartitioned) {
+ // Once per format a format table may hold. The name decides everything between the write and
+ // the read - which writer builds the file, what the file is called, which reader opens it,
+ // and whether the prefetching reader can drive it - and every other test here writes parquet,
+ // so this is where orc's path is walked end to end.
+ for (const char* file_format : {"parquet", "orc"}) {
+ SCOPED_TRACE(file_format);
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> table,
+ CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"},
+ {{Options::FILE_FORMAT, file_format}}));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first,
+ MakeBatch({1}, {"alice"}, "20240101", {{"dt", "20240101"}}));
+ ASSERT_OK(WriteAndCommit(table, std::move(first)));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second,
+ MakeBatch({2}, {"bob"}, "20240102", {{"dt", "20240102"}}));
+ ASSERT_OK(WriteAndCommit(table, std::move(second)));
+
+ // Each partition is its own directory, so each is its own split.
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatTableScan> scan,
+ FormatTableScan::Create(table, /*partition_filter=*/{}, /*limit=*/std::nullopt));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan, scan->CreatePlan());
+ ASSERT_EQ(plan->Splits().size(), 2);
+
+ // The file carries the format in its name, which is what a scan of the directory finds
+ // and what another engine reads it as.
+ std::shared_ptr<FormatDataSplit> planned =
+ std::dynamic_pointer_cast<FormatDataSplit>(plan->Splits()[0]);
+ ASSERT_TRUE(planned != nullptr);
+ ASSERT_EQ(planned->files.size(), 1u);
+ ASSERT_TRUE(
+ StringUtils::EndsWith(planned->files[0].file_path, std::string(".") + file_format))
+ << planned->files[0].file_path;
+
+ // The partition column is rebuilt from the directory name, not read from the file.
+ ASSERT_OK_AND_ASSIGN(std::vector<std::string> rows, ReadAll(table, plan->Splits()));
+ ASSERT_EQ(rows.size(), 2);
+ ASSERT_EQ(rows[0], "1|alice|20240101");
+ ASSERT_EQ(rows[1], "2|bob|20240102");
+
+ ASSERT_OK_AND_ASSIGN(PartitionList partitions, scan->ListPartitions());
+ ASSERT_EQ(partitions.size(), 2);
+ }
+}
+
+TEST(FormatTableTest, TestAFileIsReadByTheColumnsItHolds) {
+ // A format table's files need not have been written by this table. An engine that stored the
+ // columns in another order, and a file written before a column was added, are both ordinary
+ // content of such a directory: the columns are matched by name, read in the order the file
+ // holds them, and a column the file does not hold reads back as null. Both formats care -
+ // parquet refuses a column it cannot find, and orc reads its columns by their position in the
+ // file - so both are walked.
+ for (const char* file_format : {"parquet", "orc"}) {
+ SCOPED_TRACE(file_format);
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ const std::map<std::string, std::string> options = {{Options::TYPE, "format-table"},
+ {Options::FILE_FORMAT, file_format}};
+
+ // The file is written through a table of its own, whose schema is what that other engine
+ // stored: `name` in front of `id`, and no `note` at all.
+ const std::string written_path = dir->Str() + "/written";
+ std::shared_ptr<arrow::Schema> written_schema = arrow::schema(
+ {arrow::field("name", arrow::utf8()), arrow::field("id", arrow::int32())});
+ SchemaManager written_manager(dir->GetFileSystem(), written_path);
+ ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr<TableSchema> written_table_schema,
+ written_manager.CreateTable(written_schema, /*partition_keys=*/{},
+ /*primary_keys=*/{}, options));
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<FormatTable> writer_table,
+ FormatTable::Create(dir->GetFileSystem(), written_path, Identifier("db", "written"),
+ /*dynamic_options=*/{}));
+
+ auto make_written_batch = [&written_schema]() -> Result<std::unique_ptr<RecordBatch>> {
+ arrow::StringBuilder name_builder;
+ arrow::Int32Builder id_builder;
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(name_builder.Append("alice"));
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(id_builder.Append(1));
+ std::shared_ptr<arrow::Array> name_array;
+ std::shared_ptr<arrow::Array> id_array;
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(name_builder.Finish(&name_array));
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(id_builder.Finish(&id_array));
+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+ std::shared_ptr<arrow::StructArray> struct_array,
+ arrow::StructArray::Make({name_array, id_array}, written_schema->fields()));
+ auto c_array = std::make_unique<ArrowArray>();
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*struct_array, c_array.get()));
+ RecordBatchBuilder builder(c_array.get());
+ builder.SetPartition({});
+ return builder.Finish();
+ };
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch, make_written_batch());
+ ASSERT_OK(WriteAndCommit(writer_table, std::move(batch)));
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatTableScan> written_scan,
+ FormatTableScan::Create(writer_table, /*partition_filter=*/{}, /*limit=*/std::nullopt));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> written_plan, written_scan->CreatePlan());
+ ASSERT_EQ(written_plan->Splits().size(), 1u);
+ std::shared_ptr<FormatDataSplit> written_split =
+ std::dynamic_pointer_cast<FormatDataSplit>(written_plan->Splits()[0]);
+ ASSERT_TRUE(written_split != nullptr);
+ ASSERT_EQ(written_split->files.size(), 1u);
+ const std::string written_file = written_split->files[0].file_path;
+
+ // The table doing the reading declares three columns, in an order of its own.
+ const std::string read_path = dir->Str() + "/read";
+ std::shared_ptr<arrow::Schema> table_schema =
+ arrow::schema({arrow::field("id", arrow::int32()), arrow::field("name", arrow::utf8()),
+ arrow::field("note", arrow::utf8())});
+ SchemaManager read_manager(dir->GetFileSystem(), read_path);
+ ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr<TableSchema> read_table_schema,
+ read_manager.CreateTable(table_schema, /*partition_keys=*/{},
+ /*primary_keys=*/{}, options));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> table,
+ FormatTable::Create(dir->GetFileSystem(), read_path,
+ Identifier("db", "tbl"), /*dynamic_options=*/{}));
+ // The same file, now sitting in that table's directory: what an external table looks like.
+ std::error_code copied;
+ std::filesystem::copy_file(
+ written_file, PathUtil::JoinPath(read_path, PathUtil::GetName(written_file)), copied);
+ ASSERT_FALSE(copied) << copied.message();
+
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatTableScan> scan,
+ FormatTableScan::Create(table, /*partition_filter=*/{}, /*limit=*/std::nullopt));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan, scan->CreatePlan());
+ ASSERT_EQ(plan->Splits().size(), 1u);
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FormatTableRead> read,
+ FormatTableRead::TEST_Create(table, /*projection=*/std::nullopt,
+ /*pool=*/nullptr, /*predicate=*/nullptr,
+ /*enable_predicate_filter=*/false));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<BatchReader> reader,
+ read->CreateReader(plan->Splits()));
+ ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch read_batch, reader->NextBatch());
+ ASSERT_FALSE(BatchReader::IsEofBatch(read_batch));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::RecordBatch> record_batch,
+ ImportBatch(read_batch));
+ ASSERT_EQ(record_batch->num_rows(), 1);
+ // The table's own columns, in the order it declares them, behind `_VALUE_KIND`.
+ ASSERT_EQ(record_batch->schema()->field(1)->name(), "id");
+ ASSERT_EQ(record_batch->schema()->field(2)->name(), "name");
+ ASSERT_EQ(record_batch->schema()->field(3)->name(), "note");
+ ASSERT_EQ(checked_pointer_cast<arrow::Int32Array>(record_batch->column(1))->Value(0), 1);
+ ASSERT_EQ(checked_pointer_cast<arrow::StringArray>(record_batch->column(2))->GetString(0),
+ "alice");
+ // The file was written before `note` was a column of anything, so it has no value to give.
+ ASSERT_TRUE(record_batch->column(3)->IsNull(0));
+
+ ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch eof, reader->NextBatch());
+ ASSERT_TRUE(BatchReader::IsEofBatch(eof));
+ reader->Close();
+ }
+}
+
+TEST(FormatTableTest, TestAFileWhoseColumnTypeIsNotTheTablesIsRefused) {
+ // A file storing a column as a type the table does not declare is refused rather than read at
+ // the file's type. A `DECIMAL` of another scale is the case that would otherwise slip through;
+ // `DescribeFileColumns()` says why. Both formats are walked, since only orc catches it itself.
+ for (const char* file_format : {"parquet", "orc"}) {
+ SCOPED_TRACE(file_format);
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ const std::map<std::string, std::string> options = {{Options::TYPE, "format-table"},
+ {Options::FILE_FORMAT, file_format}};
+
+ // The file is written through a table of its own, which stores `amount` to two places.
+ const std::string written_path = dir->Str() + "/written";
+ std::shared_ptr<arrow::Schema> written_schema = arrow::schema(
+ {arrow::field("id", arrow::int32()), arrow::field("amount", arrow::decimal128(10, 2))});
+ SchemaManager written_manager(dir->GetFileSystem(), written_path);
+ ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr<TableSchema> written_table_schema,
+ written_manager.CreateTable(written_schema, /*partition_keys=*/{},
+ /*primary_keys=*/{}, options));
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<FormatTable> writer_table,
+ FormatTable::Create(dir->GetFileSystem(), written_path, Identifier("db", "written"),
+ /*dynamic_options=*/{}));
+
+ auto make_written_batch = [&written_schema]() -> Result<std::unique_ptr<RecordBatch>> {
+ arrow::Int32Builder id_builder;
+ arrow::Decimal128Builder amount_builder(arrow::decimal128(10, 2));
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(id_builder.Append(1));
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(amount_builder.Append(arrow::Decimal128(1234)));
+ std::shared_ptr<arrow::Array> id_array;
+ std::shared_ptr<arrow::Array> amount_array;
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(id_builder.Finish(&id_array));
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(amount_builder.Finish(&amount_array));
+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+ std::shared_ptr<arrow::StructArray> struct_array,
+ arrow::StructArray::Make({id_array, amount_array}, written_schema->fields()));
+ auto c_array = std::make_unique<ArrowArray>();
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*struct_array, c_array.get()));
+ RecordBatchBuilder builder(c_array.get());
+ builder.SetPartition({});
+ return builder.Finish();
+ };
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch, make_written_batch());
+ ASSERT_OK(WriteAndCommit(writer_table, std::move(batch)));
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatTableScan> written_scan,
+ FormatTableScan::Create(writer_table, /*partition_filter=*/{}, /*limit=*/std::nullopt));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> written_plan, written_scan->CreatePlan());
+ ASSERT_EQ(written_plan->Splits().size(), 1u);
+ std::shared_ptr<FormatDataSplit> written_split =
+ std::dynamic_pointer_cast<FormatDataSplit>(written_plan->Splits()[0]);
+ ASSERT_TRUE(written_split != nullptr);
+ ASSERT_EQ(written_split->files.size(), 1u);
+ const std::string written_file = written_split->files[0].file_path;
+
+ // The table doing the reading declares that column to three places, which is a type of its
+ // own however alike the two look.
+ const std::string read_path = dir->Str() + "/read";
+ std::shared_ptr<arrow::Schema> table_schema = arrow::schema(
+ {arrow::field("id", arrow::int32()), arrow::field("amount", arrow::decimal128(10, 3))});
+ SchemaManager read_manager(dir->GetFileSystem(), read_path);
+ ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr<TableSchema> read_table_schema,
+ read_manager.CreateTable(table_schema, /*partition_keys=*/{},
+ /*primary_keys=*/{}, options));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> table,
+ FormatTable::Create(dir->GetFileSystem(), read_path,
+ Identifier("db", "tbl"), /*dynamic_options=*/{}));
+ std::error_code copied;
+ std::filesystem::copy_file(
+ written_file, PathUtil::JoinPath(read_path, PathUtil::GetName(written_file)), copied);
+ ASSERT_FALSE(copied) << copied.message();
+
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatTableScan> scan,
+ FormatTableScan::Create(table, /*partition_filter=*/{}, /*limit=*/std::nullopt));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan, scan->CreatePlan());
+ ASSERT_EQ(plan->Splits().size(), 1u);
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FormatTableRead> read,
+ FormatTableRead::TEST_Create(table, /*projection=*/std::nullopt,
+ /*pool=*/nullptr, /*predicate=*/nullptr,
+ /*enable_predicate_filter=*/false));
+ // A split's file is opened as it is reached, so the refusal arrives with the first batch,
+ // naming the file it came from.
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<BatchReader> reader,
+ read->CreateReader(plan->Splits()));
+ ASSERT_NOK_WITH_MSG(reader->NextBatch(), "stores the column 'amount' as");
+ reader->Close();
+
+ // The same file read without that column: a projection that leaves a column out never
+ // asks the file for it, so its type is not checked.
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FormatTableRead> projected,
+ FormatTableRead::TEST_Create(table, std::vector<std::string>{"id"},
+ /*pool=*/nullptr, /*predicate=*/nullptr,
+ /*enable_predicate_filter=*/false));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<BatchReader> projected_reader,
+ projected->CreateReader(plan->Splits()));
+ ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch projected_batch, projected_reader->NextBatch());
+ ASSERT_FALSE(BatchReader::IsEofBatch(projected_batch));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::RecordBatch> projected_record,
+ ImportBatch(projected_batch));
+ ASSERT_EQ(projected_record->num_rows(), 1);
+ ASSERT_EQ(checked_pointer_cast<arrow::Int32Array>(projected_record->column(1))->Value(0),
+ 1);
+ projected_reader->Close();
+ }
+}
+
+TEST(FormatTableTest, TestAPartitionColumnStoredInTheFileNeedNotMatchTheTable) {
+ // A partition column's type in the file is never checked, the value coming from the directory
+ // name instead; `DescribeFileColumns()` says why. Here an engine wrote `dt` into the file as
+ // an `INT` while the table declares it `STRING`.
+ for (const char* file_format : {"parquet", "orc"}) {
+ SCOPED_TRACE(file_format);
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ const std::map<std::string, std::string> options = {{Options::TYPE, "format-table"},
+ {Options::FILE_FORMAT, file_format}};
+
+ const std::string written_path = dir->Str() + "/written";
+ std::shared_ptr<arrow::Schema> written_schema =
+ arrow::schema({arrow::field("id", arrow::int32()), arrow::field("name", arrow::utf8()),
+ arrow::field("dt", arrow::int32())});
+ SchemaManager written_manager(dir->GetFileSystem(), written_path);
+ ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr<TableSchema> written_table_schema,
+ written_manager.CreateTable(written_schema, /*partition_keys=*/{},
+ /*primary_keys=*/{}, options));
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<FormatTable> writer_table,
+ FormatTable::Create(dir->GetFileSystem(), written_path, Identifier("db", "written"),
+ /*dynamic_options=*/{}));
+
+ auto make_written_batch = [&written_schema]() -> Result<std::unique_ptr<RecordBatch>> {
+ arrow::Int32Builder id_builder;
+ arrow::StringBuilder name_builder;
+ arrow::Int32Builder dt_builder;
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(id_builder.Append(1));
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(name_builder.Append("alice"));
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(dt_builder.Append(20240101));
+ std::shared_ptr<arrow::Array> id_array;
+ std::shared_ptr<arrow::Array> name_array;
+ std::shared_ptr<arrow::Array> dt_array;
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(id_builder.Finish(&id_array));
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(name_builder.Finish(&name_array));
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(dt_builder.Finish(&dt_array));
+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+ std::shared_ptr<arrow::StructArray> struct_array,
+ arrow::StructArray::Make({id_array, name_array, dt_array},
+ written_schema->fields()));
+ auto c_array = std::make_unique<ArrowArray>();
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*struct_array, c_array.get()));
+ RecordBatchBuilder builder(c_array.get());
+ builder.SetPartition({});
+ return builder.Finish();
+ };
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch, make_written_batch());
+ ASSERT_OK(WriteAndCommit(writer_table, std::move(batch)));
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatTableScan> written_scan,
+ FormatTableScan::Create(writer_table, /*partition_filter=*/{}, /*limit=*/std::nullopt));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> written_plan, written_scan->CreatePlan());
+ ASSERT_EQ(written_plan->Splits().size(), 1u);
+ std::shared_ptr<FormatDataSplit> written_split =
+ std::dynamic_pointer_cast<FormatDataSplit>(written_plan->Splits()[0]);
+ ASSERT_TRUE(written_split != nullptr);
+ ASSERT_EQ(written_split->files.size(), 1u);
+ const std::string written_file = written_split->files[0].file_path;
+
+ // The reading table is partitioned by `dt`, which it declares `STRING`.
+ const std::string read_path = dir->Str() + "/read";
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> table,
+ CreateTable(dir->GetFileSystem(), read_path, {"dt"},
+ {{Options::FILE_FORMAT, file_format}}));
+ const std::string partition_dir = PathUtil::JoinPath(read_path, "dt=20240101");
+ ASSERT_OK(dir->GetFileSystem()->Mkdirs(partition_dir));
+ std::error_code copied;
+ std::filesystem::copy_file(
+ written_file, PathUtil::JoinPath(partition_dir, PathUtil::GetName(written_file)),
+ copied);
+ ASSERT_FALSE(copied) << copied.message();
+
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatTableScan> scan,
+ FormatTableScan::Create(table, /*partition_filter=*/{}, /*limit=*/std::nullopt));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan, scan->CreatePlan());
+ ASSERT_EQ(plan->Splits().size(), 1u);
+ // The directory decides the value, so it comes back as the `STRING` the table declares.
+ ASSERT_OK_AND_ASSIGN(std::vector<std::string> rows, ReadAll(table, plan->Splits()));
+ ASSERT_EQ(rows, (std::vector<std::string>{"1|alice|20240101"}));
+ }
+}
+
+TEST(FormatTableTest, TestAStructWhoseChildrenTheFileOrdersDifferentlyIsRefused) {
+ // A `STRUCT` whose children the file orders differently is refused rather than silently
+ // reordered; `FileTypeMatchesTableType()` says why. Both formats are walked: the check runs
+ // on the schema the file reports, before either reader is given one to read with, so it is
+ // this refusal that either of them meets.
+ for (const char* file_format : {"parquet", "orc"}) {
+ SCOPED_TRACE(file_format);
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ const std::map<std::string, std::string> options = {{Options::TYPE, "format-table"},
+ {Options::FILE_FORMAT, file_format}};
+
+ const std::string written_path = dir->Str() + "/written";
+ std::shared_ptr<arrow::DataType> written_payload =
+ arrow::struct_({arrow::field("b", arrow::int32()), arrow::field("a", arrow::int32())});
+ std::shared_ptr<arrow::Schema> written_schema = arrow::schema(
+ {arrow::field("id", arrow::int32()), arrow::field("payload", written_payload)});
+ SchemaManager written_manager(dir->GetFileSystem(), written_path);
+ ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr<TableSchema> written_table_schema,
+ written_manager.CreateTable(written_schema, /*partition_keys=*/{},
+ /*primary_keys=*/{}, options));
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<FormatTable> writer_table,
+ FormatTable::Create(dir->GetFileSystem(), written_path, Identifier("db", "written"),
+ /*dynamic_options=*/{}));
+
+ auto make_written_batch = [&]() -> Result<std::unique_ptr<RecordBatch>> {
+ arrow::Int32Builder id_builder;
+ arrow::Int32Builder b_builder;
+ arrow::Int32Builder a_builder;
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(id_builder.Append(1));
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(b_builder.Append(20));
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(a_builder.Append(10));
+ std::shared_ptr<arrow::Array> id_array;
+ std::shared_ptr<arrow::Array> b_array;
+ std::shared_ptr<arrow::Array> a_array;
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(id_builder.Finish(&id_array));
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(b_builder.Finish(&b_array));
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(a_builder.Finish(&a_array));
+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+ std::shared_ptr<arrow::StructArray> payload_array,
+ arrow::StructArray::Make({b_array, a_array}, written_payload->fields()));
+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+ std::shared_ptr<arrow::StructArray> struct_array,
+ arrow::StructArray::Make({id_array, payload_array}, written_schema->fields()));
+ auto c_array = std::make_unique<ArrowArray>();
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*struct_array, c_array.get()));
+ RecordBatchBuilder builder(c_array.get());
+ builder.SetPartition({});
+ return builder.Finish();
+ };
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch, make_written_batch());
+ ASSERT_OK(WriteAndCommit(writer_table, std::move(batch)));
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatTableScan> written_scan,
+ FormatTableScan::Create(writer_table, /*partition_filter=*/{}, /*limit=*/std::nullopt));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> written_plan, written_scan->CreatePlan());
+ ASSERT_EQ(written_plan->Splits().size(), 1u);
+ std::shared_ptr<FormatDataSplit> written_split =
+ std::dynamic_pointer_cast<FormatDataSplit>(written_plan->Splits()[0]);
+ ASSERT_TRUE(written_split != nullptr);
+ ASSERT_EQ(written_split->files.size(), 1u);
+ const std::string written_file = written_split->files[0].file_path;
+
+ // The same children, in the order this table declares them.
+ const std::string read_path = dir->Str() + "/read";
+ std::shared_ptr<arrow::DataType> read_payload =
+ arrow::struct_({arrow::field("a", arrow::int32()), arrow::field("b", arrow::int32())});
+ std::shared_ptr<arrow::Schema> table_schema = arrow::schema(
+ {arrow::field("id", arrow::int32()), arrow::field("payload", read_payload)});
+ SchemaManager read_manager(dir->GetFileSystem(), read_path);
+ ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr<TableSchema> read_table_schema,
+ read_manager.CreateTable(table_schema, /*partition_keys=*/{},
+ /*primary_keys=*/{}, options));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> table,
+ FormatTable::Create(dir->GetFileSystem(), read_path,
+ Identifier("db", "tbl"), /*dynamic_options=*/{}));
+ std::error_code copied;
+ std::filesystem::copy_file(
+ written_file, PathUtil::JoinPath(read_path, PathUtil::GetName(written_file)), copied);
+ ASSERT_FALSE(copied) << copied.message();
+
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatTableScan> scan,
+ FormatTableScan::Create(table, /*partition_filter=*/{}, /*limit=*/std::nullopt));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan, scan->CreatePlan());
+ ASSERT_EQ(plan->Splits().size(), 1u);
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FormatTableRead> read,
+ FormatTableRead::TEST_Create(table, /*projection=*/std::nullopt,
+ /*pool=*/nullptr, /*predicate=*/nullptr,
+ /*enable_predicate_filter=*/false));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<BatchReader> reader,
+ read->CreateReader(plan->Splits()));
+ ASSERT_NOK_WITH_MSG(reader->NextBatch(), "stores the column 'payload' as");
+ reader->Close();
+ }
+}
+
+TEST(FormatTableTest, TestAFileMissingANotNullColumnIsRefused) {
+ // A column the file leaves out is read as null, so one the table declares NOT NULL is refused
+ // instead. See `DescribeFileColumns()`.
+ for (const char* file_format : {"parquet", "orc"}) {
+ SCOPED_TRACE(file_format);
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ const std::map<std::string, std::string> options = {{Options::TYPE, "format-table"},
+ {Options::FILE_FORMAT, file_format}};
+
+ const std::string written_path = dir->Str() + "/written";
+ std::shared_ptr<arrow::Schema> written_schema =
+ arrow::schema({arrow::field("id", arrow::int32())});
+ SchemaManager written_manager(dir->GetFileSystem(), written_path);
+ ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr<TableSchema> written_table_schema,
+ written_manager.CreateTable(written_schema, /*partition_keys=*/{},
+ /*primary_keys=*/{}, options));
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<FormatTable> writer_table,
+ FormatTable::Create(dir->GetFileSystem(), written_path, Identifier("db", "written"),
+ /*dynamic_options=*/{}));
+
+ auto make_written_batch = [&written_schema]() -> Result<std::unique_ptr<RecordBatch>> {
+ arrow::Int32Builder id_builder;
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(id_builder.Append(1));
+ std::shared_ptr<arrow::Array> id_array;
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(id_builder.Finish(&id_array));
+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+ std::shared_ptr<arrow::StructArray> struct_array,
+ arrow::StructArray::Make({id_array}, written_schema->fields()));
+ auto c_array = std::make_unique<ArrowArray>();
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*struct_array, c_array.get()));
+ RecordBatchBuilder builder(c_array.get());
+ builder.SetPartition({});
+ return builder.Finish();
+ };
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch, make_written_batch());
+ ASSERT_OK(WriteAndCommit(writer_table, std::move(batch)));
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatTableScan> written_scan,
+ FormatTableScan::Create(writer_table, /*partition_filter=*/{}, /*limit=*/std::nullopt));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> written_plan, written_scan->CreatePlan());
+ ASSERT_EQ(written_plan->Splits().size(), 1u);
+ std::shared_ptr<FormatDataSplit> written_split =
+ std::dynamic_pointer_cast<FormatDataSplit>(written_plan->Splits()[0]);
+ ASSERT_TRUE(written_split != nullptr);
+ ASSERT_EQ(written_split->files.size(), 1u);
+ const std::string written_file = written_split->files[0].file_path;
+
+ // The reading table declares a column the file has no value to give, and refuses null for
+ // it.
+ const std::string read_path = dir->Str() + "/read";
+ std::shared_ptr<arrow::Schema> table_schema =
+ arrow::schema({arrow::field("id", arrow::int32()),
+ arrow::field("note", arrow::utf8(), /*nullable=*/false)});
+ SchemaManager read_manager(dir->GetFileSystem(), read_path);
+ ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr<TableSchema> read_table_schema,
+ read_manager.CreateTable(table_schema, /*partition_keys=*/{},
+ /*primary_keys=*/{}, options));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> table,
+ FormatTable::Create(dir->GetFileSystem(), read_path,
+ Identifier("db", "tbl"), /*dynamic_options=*/{}));
+ std::error_code copied;
+ std::filesystem::copy_file(
+ written_file, PathUtil::JoinPath(read_path, PathUtil::GetName(written_file)), copied);
+ ASSERT_FALSE(copied) << copied.message();
+
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatTableScan> scan,
+ FormatTableScan::Create(table, /*partition_filter=*/{}, /*limit=*/std::nullopt));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan, scan->CreatePlan());
+ ASSERT_EQ(plan->Splits().size(), 1u);
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FormatTableRead> read,
+ FormatTableRead::TEST_Create(table, /*projection=*/std::nullopt,
+ /*pool=*/nullptr, /*predicate=*/nullptr,
+ /*enable_predicate_filter=*/false));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<BatchReader> reader,
+ read->CreateReader(plan->Splits()));
+ ASSERT_NOK_WITH_MSG(reader->NextBatch(), "which the table declares NOT NULL");
+ reader->Close();
+
+ // A projection that leaves it out asks the file for nothing it does not hold.
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FormatTableRead> projected,
+ FormatTableRead::TEST_Create(table, std::vector<std::string>{"id"},
+ /*pool=*/nullptr, /*predicate=*/nullptr,
+ /*enable_predicate_filter=*/false));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<BatchReader> projected_reader,
+ projected->CreateReader(plan->Splits()));
+ ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch projected_batch, projected_reader->NextBatch());
+ ASSERT_FALSE(BatchReader::IsEofBatch(projected_batch));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::RecordBatch> projected_record,
+ ImportBatch(projected_batch));
+ ASSERT_EQ(projected_record->num_rows(), 1);
+ projected_reader->Close();
+ }
+}
+
+namespace {
+
+/// The schema the timestamp tests below use: an `id` and a `ts` in `unit`.
+std::shared_ptr<arrow::Schema> MakeTimestampSchema(arrow::TimeUnit::type unit) {
+ return arrow::schema(
+ {arrow::field("id", arrow::int32()), arrow::field("ts", arrow::timestamp(unit))});
+}
+
+/// Creates a format table of `file_format` under `path`, its `ts` column declared in `unit`.
+Result<std::shared_ptr<FormatTable>> CreateTimestampTable(
+ const std::shared_ptr<FileSystem>& file_system, const std::string& path,
+ const std::string& file_format, arrow::TimeUnit::type unit) {
+ SchemaManager manager(file_system, path);
+ PAIMON_ASSIGN_OR_RAISE(
+ [[maybe_unused]] std::unique_ptr<TableSchema> table_schema,
+ manager.CreateTable(
+ MakeTimestampSchema(unit), /*partition_keys=*/{}, /*primary_keys=*/{},
+ {{Options::TYPE, "format-table"}, {Options::FILE_FORMAT, file_format}}));
+ return FormatTable::Create(file_system, path, Identifier("db", PathUtil::GetName(path)),
+ /*dynamic_options=*/{});
+}
+
+/// Builds a one-row batch of `MakeTimestampSchema(unit)` whose `ts` holds `value`, counted in
+/// `unit`.
+Result<std::unique_ptr<RecordBatch>> MakeTimestampBatch(arrow::TimeUnit::type unit, int64_t value) {
+ std::shared_ptr<arrow::Schema> schema = MakeTimestampSchema(unit);
+ arrow::Int32Builder id_builder;
+ arrow::TimestampBuilder ts_builder(schema->field(1)->type(), arrow::default_memory_pool());
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(id_builder.Append(1));
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(ts_builder.Append(value));
+ std::shared_ptr<arrow::Array> id_array;
+ std::shared_ptr<arrow::Array> ts_array;
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(id_builder.Finish(&id_array));
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(ts_builder.Finish(&ts_array));
+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+ std::shared_ptr<arrow::StructArray> struct_array,
+ arrow::StructArray::Make({id_array, ts_array}, schema->fields()));
+ auto c_array = std::make_unique<ArrowArray>();
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*struct_array, c_array.get()));
+ RecordBatchBuilder builder(c_array.get());
+ builder.SetPartition({});
+ return builder.Finish();
+}
+
+/// One `ts` value a read returned, with the unit it came back in.
+struct TimestampRow {
+ arrow::TimeUnit::type unit;
+ int64_t value;
+};
+
+/// One row read whole, with the reader that produced it: a batch borrows memory from its reader,
+/// so a caller reads its values out before letting this go.
+struct OneRowRead {
+ std::unique_ptr<BatchReader> reader;
+ std::shared_ptr<arrow::RecordBatch> batch;
+};
+
+/// Reads the one row `table` holds, checking that the reader runs out after it - otherwise the
+/// row asserted on would be no more than the first of several.
+Result<OneRowRead> ReadOneRow(const std::shared_ptr<FormatTable>& table) {
+ PAIMON_ASSIGN_OR_RAISE(
+ std::unique_ptr<FormatTableScan> scan,
+ FormatTableScan::Create(table, /*partition_filter=*/{}, /*limit=*/std::nullopt));
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Plan> plan, scan->CreatePlan());
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FormatTableRead> read,
+ FormatTableRead::TEST_Create(table, /*projection=*/std::nullopt,
+ /*pool=*/nullptr, /*predicate=*/nullptr,
+ /*enable_predicate_filter=*/false));
+ OneRowRead result;
+ PAIMON_ASSIGN_OR_RAISE(result.reader, read->CreateReader(plan->Splits()));
+ PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, result.reader->NextBatch());
+ if (BatchReader::IsEofBatch(batch)) {
+ return Status::Invalid("the table holds no rows");
+ }
+ PAIMON_ASSIGN_OR_RAISE(result.batch, ImportBatch(batch));
+ if (result.batch->num_rows() != 1) {
+ return Status::Invalid("the table holds more than the one row that was written");
+ }
+ PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch next, result.reader->NextBatch());
+ if (!BatchReader::IsEofBatch(next)) {
+ return Status::Invalid("the table holds more than the one batch that was written");
+ }
+ return result;
+}
+
+/// The unit and value of a timestamp column a read returned. Both are the point: a value returned
+/// under the wrong unit is off by a factor of a thousand and nothing else in the batch says so.
+Result<TimestampRow> ToTimestampRow(const std::shared_ptr<arrow::Array>& column) {
+ if (column == nullptr || column->type_id() != arrow::Type::TIMESTAMP) {
+ return Status::Invalid("the read returned no timestamp column");
+ }
+ return TimestampRow{checked_pointer_cast<arrow::TimestampType>(column->type())->unit(),
+ checked_pointer_cast<arrow::TimestampArray>(column)->Value(0)};
+}
+
+/// The `ts` of the one row `table` holds.
+Result<TimestampRow> ReadOneTimestamp(const std::shared_ptr<FormatTable>& table) {
+ PAIMON_ASSIGN_OR_RAISE(OneRowRead read, ReadOneRow(table));
+ PAIMON_ASSIGN_OR_RAISE(TimestampRow row, ToTimestampRow(read.batch->GetColumnByName("ts")));
+ read.reader->Close();
+ return row;
+}
+
+/// The `payload.ts` of the one row `table` holds, which reaches the type check through its
+/// recursive half rather than at the top level.
+Result<TimestampRow> ReadOneNestedTimestamp(const std::shared_ptr<FormatTable>& table) {
+ PAIMON_ASSIGN_OR_RAISE(OneRowRead read, ReadOneRow(table));
+ std::shared_ptr<arrow::Array> payload = read.batch->GetColumnByName("payload");
+ if (payload == nullptr || payload->type_id() != arrow::Type::STRUCT) {
+ return Status::Invalid("the read returned no struct column named 'payload'");
+ }
+ PAIMON_ASSIGN_OR_RAISE(
+ TimestampRow row,
+ ToTimestampRow(checked_pointer_cast<arrow::StructArray>(payload)->field(0)));
+ read.reader->Close();
+ return row;
+}
+
+/// The nested counterpart of `MakeTimestampSchema()`: the `ts` sits inside a `payload` struct.
+std::shared_ptr<arrow::Schema> MakeNestedTimestampSchema(arrow::TimeUnit::type unit) {
+ return arrow::schema(
+ {arrow::field("id", arrow::int32()),
+ arrow::field("payload", arrow::struct_({arrow::field("ts", arrow::timestamp(unit))}))});
+}
+
+/// `CreateTimestampTable()` for `MakeNestedTimestampSchema()`.
+Result<std::shared_ptr<FormatTable>> CreateNestedTimestampTable(
+ const std::shared_ptr<FileSystem>& file_system, const std::string& path,
+ const std::string& file_format, arrow::TimeUnit::type unit) {
+ SchemaManager manager(file_system, path);
+ PAIMON_ASSIGN_OR_RAISE(
+ [[maybe_unused]] std::unique_ptr<TableSchema> table_schema,
+ manager.CreateTable(
+ MakeNestedTimestampSchema(unit), /*partition_keys=*/{}, /*primary_keys=*/{},
+ {{Options::TYPE, "format-table"}, {Options::FILE_FORMAT, file_format}}));
+ return FormatTable::Create(file_system, path, Identifier("db", PathUtil::GetName(path)),
+ /*dynamic_options=*/{});
+}
+
+/// `MakeTimestampBatch()` for `MakeNestedTimestampSchema()`.
+Result<std::unique_ptr<RecordBatch>> MakeNestedTimestampBatch(arrow::TimeUnit::type unit,
+ int64_t value) {
+ std::shared_ptr<arrow::Schema> schema = MakeNestedTimestampSchema(unit);
+ arrow::Int32Builder id_builder;
+ arrow::TimestampBuilder ts_builder(arrow::timestamp(unit), arrow::default_memory_pool());
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(id_builder.Append(1));
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(ts_builder.Append(value));
+ std::shared_ptr<arrow::Array> id_array;
+ std::shared_ptr<arrow::Array> ts_array;
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(id_builder.Finish(&id_array));
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(ts_builder.Finish(&ts_array));
+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+ std::shared_ptr<arrow::StructArray> payload_array,
+ arrow::StructArray::Make({ts_array}, schema->field(1)->type()->fields()));
+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+ std::shared_ptr<arrow::StructArray> struct_array,
+ arrow::StructArray::Make({id_array, payload_array}, schema->fields()));
+ auto c_array = std::make_unique<ArrowArray>();
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*struct_array, c_array.get()));
+ RecordBatchBuilder builder(c_array.get());
+ builder.SetPartition({});
+ return builder.Finish();
+}
+
+/// The one data file `table` holds, as its own plan names it.
+Result<std::string> OneDataFilePath(const std::shared_ptr<FormatTable>& table) {
+ PAIMON_ASSIGN_OR_RAISE(
+ std::unique_ptr<FormatTableScan> scan,
+ FormatTableScan::Create(table, /*partition_filter=*/{}, /*limit=*/std::nullopt));
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Plan> plan, scan->CreatePlan());
+ if (plan->Splits().size() != 1) {
+ return Status::Invalid("the table holds more than the one split that was written");
+ }
+ std::shared_ptr<FormatDataSplit> split =
+ std::dynamic_pointer_cast<FormatDataSplit>(plan->Splits()[0]);
+ if (split == nullptr || split->files.size() != 1) {
+ return Status::Invalid("the table holds more than the one file that was written");
+ }
+ return split->files[0].file_path;
+}
+
+} // namespace
+
+TEST(FormatTableTest, TestAParquetTimestampOfAnotherUnitIsRefusedButTheWritersOwnPairIsNot) {
+ // A `SECOND` column survives parquet's round trip through `MILLI`, and a `us` file under an
+ // `ms` table is refused rather than relabelled. `TimestampUnitIsCompatible()` says why.
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> second_table,
+ CreateTimestampTable(dir->GetFileSystem(), dir->Str() + "/second",
+ "parquet", arrow::TimeUnit::SECOND));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
+ MakeTimestampBatch(arrow::TimeUnit::SECOND, 1700000000));
+ ASSERT_OK(WriteAndCommit(second_table, std::move(second_batch)));
+ ASSERT_OK_AND_ASSIGN(TimestampRow second_row, ReadOneTimestamp(second_table));
+ ASSERT_EQ(second_row.unit, arrow::TimeUnit::SECOND);
+ ASSERT_EQ(second_row.value, 1700000000);
+
+ // The same instant written in `us`, whose file the `ms` table below is then handed.
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> micro_table,
+ CreateTimestampTable(dir->GetFileSystem(), dir->Str() + "/micro",
+ "parquet", arrow::TimeUnit::MICRO));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> micro_batch,
+ MakeTimestampBatch(arrow::TimeUnit::MICRO, 1700000000123456));
+ ASSERT_OK(WriteAndCommit(micro_table, std::move(micro_batch)));
+ ASSERT_OK_AND_ASSIGN(TimestampRow micro_row, ReadOneTimestamp(micro_table));
+ ASSERT_EQ(micro_row.unit, arrow::TimeUnit::MICRO);
+ ASSERT_EQ(micro_row.value, 1700000000123456);
+
+ const std::string milli_path = dir->Str() + "/milli";
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<FormatTable> milli_table,
+ CreateTimestampTable(dir->GetFileSystem(), milli_path, "parquet", arrow::TimeUnit::MILLI));
+ ASSERT_OK_AND_ASSIGN(std::string micro_file, OneDataFilePath(micro_table));
+ std::error_code copied;
+ std::filesystem::copy_file(
+ micro_file, PathUtil::JoinPath(milli_path, PathUtil::GetName(micro_file)), copied);
+ ASSERT_FALSE(copied) << copied.message();
+ ASSERT_NOK_WITH_MSG(ReadOneTimestamp(milli_table), "stores the column 'ts' as");
+}
+
+TEST(FormatTableTest, TestANestedTimestampFollowsTheSameRuleAsATopLevelOne) {
+ // The type check walks into a `STRUCT`, so the per-format unit rule has to hold at depth
+ // too, and a refusal has to travel back up out of the recursion.
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> orc_table,
+ CreateNestedTimestampTable(dir->GetFileSystem(), dir->Str() + "/orc",
+ "orc", arrow::TimeUnit::MILLI));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> orc_batch,
+ MakeNestedTimestampBatch(arrow::TimeUnit::MILLI, 1700000000123));
+ ASSERT_OK(WriteAndCommit(orc_table, std::move(orc_batch)));
+ ASSERT_OK_AND_ASSIGN(TimestampRow orc_row, ReadOneNestedTimestamp(orc_table));
+ ASSERT_EQ(orc_row.unit, arrow::TimeUnit::MILLI);
+ ASSERT_EQ(orc_row.value, 1700000000123);
+
+ // parquet keeps the unit it wrote, so a nested `us` under a nested `ms` column is refused as
+ // a top-level one is - the recursion carries the refusal back up.
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> micro_table,
+ CreateNestedTimestampTable(dir->GetFileSystem(), dir->Str() + "/micro",
+ "parquet", arrow::TimeUnit::MICRO));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> micro_batch,
+ MakeNestedTimestampBatch(arrow::TimeUnit::MICRO, 1700000000123456));
+ ASSERT_OK(WriteAndCommit(micro_table, std::move(micro_batch)));
+ ASSERT_OK_AND_ASSIGN(TimestampRow micro_row, ReadOneNestedTimestamp(micro_table));
+ ASSERT_EQ(micro_row.unit, arrow::TimeUnit::MICRO);
+ ASSERT_EQ(micro_row.value, 1700000000123456);
+
+ const std::string milli_path = dir->Str() + "/milli";
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> milli_table,
+ CreateNestedTimestampTable(dir->GetFileSystem(), milli_path, "parquet",
+ arrow::TimeUnit::MILLI));
+ ASSERT_OK_AND_ASSIGN(std::string micro_file, OneDataFilePath(micro_table));
+ std::error_code copied;
+ std::filesystem::copy_file(
+ micro_file, PathUtil::JoinPath(milli_path, PathUtil::GetName(micro_file)), copied);
+ ASSERT_FALSE(copied) << copied.message();
+ ASSERT_NOK_WITH_MSG(ReadOneNestedTimestamp(milli_table), "stores the column 'payload' as");
+}
+
+TEST(FormatTableTest, TestAnOrcTimestampIsReadAtTheUnitTheTableDeclares) {
+ // Every unit survives orc's round trip, and a file another unit wrote is read at the table's
+ // rather than refused. Applying parquet's rule here would refuse every orc `TIMESTAMP(0)`,
+ // `(3)` and `(6)`, this library's own writes among them; `TimestampUnitIsCompatible()` says
+ // why the two formats differ.
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ const std::vector<std::tuple<const char*, arrow::TimeUnit::type, int64_t>> instants = {
+ {"second", arrow::TimeUnit::SECOND, 1700000000},
+ {"milli", arrow::TimeUnit::MILLI, 1700000000123},
+ {"micro", arrow::TimeUnit::MICRO, 1700000000123456},
+ {"nano", arrow::TimeUnit::NANO, 1700000000123456789}};
+ for (const auto& [name, unit, value] : instants) {
+ SCOPED_TRACE(name);
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<FormatTable> table,
+ CreateTimestampTable(dir->GetFileSystem(), dir->Str() + "/" + name, "orc", unit));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch, MakeTimestampBatch(unit, value));
+ ASSERT_OK(WriteAndCommit(table, std::move(batch)));
+ ASSERT_OK_AND_ASSIGN(TimestampRow row, ReadOneTimestamp(table));
+ ASSERT_EQ(row.unit, unit);
+ ASSERT_EQ(row.value, value);
+ }
+
+ // A file another unit wrote is read at the table's, rather than refused as parquet's is: the
+ // seconds and nanoseconds orc stored are rescaled, not relabelled, so the instant survives.
+ const std::string milli_path = dir->Str() + "/read_as_milli";
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<FormatTable> milli_table,
+ CreateTimestampTable(dir->GetFileSystem(), milli_path, "orc", arrow::TimeUnit::MILLI));
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<FormatTable> micro_table,
+ CreateTimestampTable(dir->GetFileSystem(), dir->Str() + "/written_as_micro", "orc",
+ arrow::TimeUnit::MICRO));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> micro_batch,
+ MakeTimestampBatch(arrow::TimeUnit::MICRO, 1700000000123456));
+ ASSERT_OK(WriteAndCommit(micro_table, std::move(micro_batch)));
+ ASSERT_OK_AND_ASSIGN(std::string micro_file, OneDataFilePath(micro_table));
+ std::error_code copied;
+ std::filesystem::copy_file(
+ micro_file, PathUtil::JoinPath(milli_path, PathUtil::GetName(micro_file)), copied);
+ ASSERT_FALSE(copied) << copied.message();
+ ASSERT_OK_AND_ASSIGN(TimestampRow milli_row, ReadOneTimestamp(milli_table));
+ ASSERT_EQ(milli_row.unit, arrow::TimeUnit::MILLI);
+ ASSERT_EQ(milli_row.value, 1700000000123);
+}
+
+TEST(FormatTableTest, TestPartitionsAreListedInAStableOrder) {
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> table,
+ CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"}));
+ // Written out of order, because the listing order must not depend on it: the file system
+ // promises none, and a caller comparing two listings would see partitions move.
+ for (const char* dt : {"20240103", "20240101", "20240102"}) {
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+ MakeBatch({1}, {"alice"}, dt, {{"dt", dt}}));
+ ASSERT_OK(WriteAndCommit(table, std::move(batch)));
+ }
+
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatTableScan> scan,
+ FormatTableScan::Create(table, /*partition_filter=*/{}, /*limit=*/std::nullopt));
+ ASSERT_OK_AND_ASSIGN(PartitionList partitions, scan->ListPartitions());
+ ASSERT_EQ(partitions.size(), 3u);
+ ASSERT_EQ(partitions[0].at("dt"), "20240101");
+ ASSERT_EQ(partitions[1].at("dt"), "20240102");
+ ASSERT_EQ(partitions[2].at("dt"), "20240103");
+}
+
+TEST(FormatTableTest, TestPartitionFilter) {
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> table,
+ CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"}));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first,
+ MakeBatch({1}, {"alice"}, "20240101", {{"dt", "20240101"}}));
+ ASSERT_OK(WriteAndCommit(table, std::move(first)));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second,
+ MakeBatch({2}, {"bob"}, "20240102", {{"dt", "20240102"}}));
+ ASSERT_OK(WriteAndCommit(table, std::move(second)));
+
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatTableScan> scan,
+ FormatTableScan::Create(table, {{"dt", "20240102"}}, /*limit=*/std::nullopt));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan, scan->CreatePlan());
+ ASSERT_EQ(plan->Splits().size(), 1);
+ ASSERT_OK_AND_ASSIGN(std::vector<std::string> rows, ReadAll(table, plan->Splits()));
+ ASSERT_EQ(rows.size(), 1);
+ ASSERT_EQ(rows[0], "2|bob|20240102");
+}
+
+TEST(FormatTableTest, TestPartitionFilterMatchesThroughTheColumnType) {
+ // A partition filter names a value, not the text a directory happens to be named with. Java
+ // tests a `PartitionPredicate` against the `BinaryRow` a directory parses into and reaches the
+ // same answer; comparing the raw strings would make `1` miss a directory called `pt=01`.
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ std::shared_ptr<arrow::Schema> int_schema =
+ arrow::schema({arrow::field("name", arrow::utf8()), arrow::field("pt", arrow::int32())});
+ SchemaManager schema_manager(dir->GetFileSystem(), dir->Str());
+ ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr<TableSchema> table_schema,
+ schema_manager.CreateTable(
+ int_schema, /*partition_keys=*/{"pt"}, /*primary_keys=*/{},
+ {{Options::TYPE, "format-table"}, {Options::FILE_FORMAT, "parquet"}}));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> table,
+ FormatTable::Create(dir->GetFileSystem(), dir->Str(),
+ Identifier("db", "tbl"), /*dynamic_options=*/{}));
+
+ // Written by some other engine, which spelled the value with a leading zero.
+ ASSERT_OK(dir->GetFileSystem()->Mkdirs(dir->Str() + "/pt=01"));
+ ASSERT_OK(dir->GetFileSystem()->Mkdirs(dir->Str() + "/pt=2"));
+
+ for (const char* spelling : {"1", "01"}) {
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<FormatTableScan> scan,
+ FormatTableScan::Create(table, {{"pt", spelling}}, /*limit=*/std::nullopt));
+ ASSERT_OK_AND_ASSIGN(PartitionList partitions, scan->ListPartitions());
+ ASSERT_EQ(partitions.size(), 1u) << spelling;
+ // The directory's own value, not the normalised one: a format table's partitions are its
+ // directories, so a plan reports what is on disk.
+ ASSERT_EQ(partitions[0].at("pt"), "01") << spelling;
+ }
+
+ // And a value no `INT` column can hold is the caller's mistake, not an empty result.
+ ASSERT_NOK_WITH_MSG(FormatTableScan::Create(table, {{"pt", "abc"}}, /*limit=*/std::nullopt),
+ "cannot be read into the partition columns");
+}
+
+TEST(FormatTableTest, TestPartitionFilterCanNameTheNullPartition) {
+ // A blank partition value stands for null and lands in the `partition.default-name`
+ // directory. Both sides of the filter are read into the column type, where that name is null,
+ // so naming it selects that partition and nothing else.
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> table,
+ CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"}));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> blank,
+ MakeBatch({1}, {"alice"}, " ", {{"dt", table->PartitionDefaultName()}}));
+ ASSERT_OK(WriteAndCommit(table, std::move(blank)));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> dated,
+ MakeBatch({2}, {"bob"}, "20240101", {{"dt", "20240101"}}));
+ ASSERT_OK(WriteAndCommit(table, std::move(dated)));
+
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FormatTableScan> scan,
+ FormatTableScan::Create(table, {{"dt", table->PartitionDefaultName()}},
+ /*limit=*/std::nullopt));
+ ASSERT_OK_AND_ASSIGN(PartitionList partitions, scan->ListPartitions());
+ ASSERT_EQ(partitions, (PartitionList{{{"dt", table->PartitionDefaultName()}}}));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan, scan->CreatePlan());
+ ASSERT_OK_AND_ASSIGN(std::vector<std::string> rows, ReadAll(table, plan->Splits()));
+ ASSERT_EQ(rows.size(), 1u);
+}
+
+TEST(FormatTableTest, TestABlankPartitionDirectoryIsNotTheNullPartition) {
+ // A write of a blank value lands in the null partition's directory, but a directory that holds
+ // a blank value is still not the null partition: the two are told apart by what the column
+ // holds, not by the directory name a write would have chosen for it, as Java tests its
+ // `PartitionPredicate` against the parsed value. Rendering both sides back out to a directory
+ // name would fold the two together, so `partition.default-name` would select a blank directory
+ // too and hand out a blank value the filter never named.
+ std::unique_ptr<UniqueTestDirectory> dir = UniqueTestDirectory::Create();
+ ASSERT_TRUE(dir);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FormatTable> table,
+ CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"}));
+ // Written by some other engine. A tab is escaped in a path, so the directory has a legal name
+ // while its value is blank.
+ ASSERT_OK(dir->GetFileSystem()->Mkdirs(dir->Str() + "/dt=%09"));
+ ASSERT_OK(dir->GetFileSystem()->Mkdirs(dir->Str() + "/dt=" + table->PartitionDefaultName()));
+
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FormatTableScan> null_scan,
... 4688 lines suppressed ...