You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by fo...@apache.org on 2022/09/01 09:34:19 UTC
[iceberg] branch master updated: [Python] FsspecFileIO that wraps any fsspec filesystem (#5332)
This is an automated email from the ASF dual-hosted git repository.
fokko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new b2c4ec2620 [Python] FsspecFileIO that wraps any fsspec filesystem (#5332)
b2c4ec2620 is described below
commit b2c4ec2620b44e2aba1962870bb2de9ecf25d5fd
Author: Samuel Redai <43...@users.noreply.github.com>
AuthorDate: Thu Sep 1 05:34:12 2022 -0400
[Python] FsspecFileIO that wraps any fsspec filesystem (#5332)
---
python/Makefile | 4 +-
python/poetry.lock | 345 ++++++++++++++++++++++++++++++++++++-----
python/pyiceberg/io/fsspec.py | 196 +++++++++++++++++++++++
python/pyproject.toml | 17 ++
python/tests/conftest.py | 21 +++
python/tests/io/test_fsspec.py | 200 ++++++++++++++++++++++++
6 files changed, 742 insertions(+), 41 deletions(-)
diff --git a/python/Makefile b/python/Makefile
index aef783db33..0a67cc7817 100644
--- a/python/Makefile
+++ b/python/Makefile
@@ -17,13 +17,13 @@
install:
pip install poetry
- poetry install -E pyarrow -E hive
+ poetry install -E pyarrow -E hive -E s3fs
lint:
poetry run pre-commit run --all-files
test:
- poetry run coverage run --source=pyiceberg/ -m pytest tests/ ${PYTEST_ARGS}
+ poetry run coverage run --source=pyiceberg/ -m pytest tests/ -m "not s3" ${PYTEST_ARGS}
poetry run coverage report -m --fail-under=90
poetry run coverage html
poetry run coverage xml
diff --git a/python/poetry.lock b/python/poetry.lock
index 45f58cf672..78aac49821 100644
--- a/python/poetry.lock
+++ b/python/poetry.lock
@@ -1,3 +1,71 @@
+[[package]]
+name = "aiobotocore"
+version = "2.3.4"
+description = "Async client for aws services using botocore and aiohttp"
+category = "main"
+optional = true
+python-versions = ">=3.6"
+
+[package.dependencies]
+aiohttp = ">=3.3.1"
+aioitertools = ">=0.5.1"
+botocore = ">=1.24.21,<1.24.22"
+wrapt = ">=1.10.10"
+
+[package.extras]
+awscli = ["awscli (>=1.22.76,<1.22.77)"]
+boto3 = ["boto3 (>=1.21.21,<1.21.22)"]
+
+[[package]]
+name = "aiohttp"
+version = "3.8.1"
+description = "Async http client/server framework (asyncio)"
+category = "main"
+optional = true
+python-versions = ">=3.6"
+
+[package.dependencies]
+aiosignal = ">=1.1.2"
+async-timeout = ">=4.0.0a3,<5.0"
+attrs = ">=17.3.0"
+charset-normalizer = ">=2.0,<3.0"
+frozenlist = ">=1.1.1"
+multidict = ">=4.5,<7.0"
+yarl = ">=1.0,<2.0"
+
+[package.extras]
+speedups = ["aiodns", "brotli", "cchardet"]
+
+[[package]]
+name = "aioitertools"
+version = "0.10.0"
+description = "itertools and builtins for AsyncIO and mixed iterables"
+category = "main"
+optional = true
+python-versions = ">=3.6"
+
+[package.dependencies]
+typing_extensions = {version = ">=4.0", markers = "python_version < \"3.10\""}
+
+[[package]]
+name = "aiosignal"
+version = "1.2.0"
+description = "aiosignal: a list of registered asynchronous callbacks"
+category = "main"
+optional = true
+python-versions = ">=3.6"
+
+[package.dependencies]
+frozenlist = ">=1.1.0"
+
+[[package]]
+name = "async-timeout"
+version = "4.0.2"
+description = "Timeout context manager for asyncio programs"
+category = "main"
+optional = true
+python-versions = ">=3.6"
+
[[package]]
name = "atomicwrites"
version = "1.4.1"
@@ -10,7 +78,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
name = "attrs"
version = "22.1.0"
description = "Classes Without Boilerplate"
-category = "dev"
+category = "main"
optional = false
python-versions = ">=3.5"
@@ -20,6 +88,22 @@ docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"]
tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "zope.interface"]
tests_no_zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"]
+[[package]]
+name = "botocore"
+version = "1.24.21"
+description = "Low-level, data-driven core of boto 3."
+category = "main"
+optional = true
+python-versions = ">= 3.6"
+
+[package.dependencies]
+jmespath = ">=0.7.1,<2.0.0"
+python-dateutil = ">=2.1,<3.0.0"
+urllib3 = ">=1.25.4,<1.27"
+
+[package.extras]
+crt = ["awscrt (==0.13.5)"]
+
[[package]]
name = "certifi"
version = "2022.6.15"
@@ -144,6 +228,45 @@ python-versions = ">=3.7"
docs = ["furo (>=2022.6.21)", "sphinx (>=5.1.1)", "sphinx-autodoc-typehints (>=1.19.1)"]
testing = ["covdefaults (>=2.2)", "coverage (>=6.4.2)", "pytest (>=7.1.2)", "pytest-cov (>=3)", "pytest-timeout (>=2.1)"]
+[[package]]
+name = "frozenlist"
+version = "1.3.1"
+description = "A list-like structure which implements collections.abc.MutableSequence"
+category = "main"
+optional = true
+python-versions = ">=3.7"
+
+[[package]]
+name = "fsspec"
+version = "2022.5.0"
+description = "File-system specification"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.extras]
+abfs = ["adlfs"]
+adl = ["adlfs"]
+arrow = ["pyarrow (>=1)"]
+dask = ["dask", "distributed"]
+dropbox = ["dropbox", "dropboxdrivefs", "requests"]
+entrypoints = ["importlib-metadata"]
+fuse = ["fusepy"]
+gcs = ["gcsfs"]
+git = ["pygit2"]
+github = ["requests"]
+gs = ["gcsfs"]
+gui = ["panel"]
+hdfs = ["pyarrow (>=1)"]
+http = ["aiohttp", "requests"]
+libarchive = ["libarchive-c"]
+oci = ["ocifs"]
+s3 = ["s3fs"]
+sftp = ["paramiko"]
+smb = ["smbprotocol"]
+ssh = ["paramiko"]
+tqdm = ["tqdm"]
+
[[package]]
name = "identify"
version = "2.5.3"
@@ -187,6 +310,14 @@ category = "dev"
optional = false
python-versions = "*"
+[[package]]
+name = "jmespath"
+version = "1.0.1"
+description = "JSON Matching Expressions"
+category = "main"
+optional = true
+python-versions = ">=3.7"
+
[[package]]
name = "mmh3"
version = "3.0.0"
@@ -195,6 +326,14 @@ category = "main"
optional = false
python-versions = "*"
+[[package]]
+name = "multidict"
+version = "6.0.2"
+description = "multidict implementation"
+category = "main"
+optional = true
+python-versions = ">=3.7"
+
[[package]]
name = "nodeenv"
version = "1.7.0"
@@ -302,7 +441,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
[[package]]
name = "pydantic"
-version = "1.10.0"
+version = "1.10.1"
description = "Data validation and settings management using python type hints"
category = "main"
optional = false
@@ -375,6 +514,17 @@ pep517 = "*"
docs = ["jaraco.packaging (>=8.2)", "rst.linker (>=1.9)", "sphinx"]
testing = ["pytest (>=4.6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-flake8", "pytest-mypy", "types-docutils"]
+[[package]]
+name = "python-dateutil"
+version = "2.8.2"
+description = "Extensions to the standard Python datetime module"
+category = "main"
+optional = true
+python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"
+
+[package.dependencies]
+six = ">=1.5"
+
[[package]]
name = "python-snappy"
version = "0.6.1"
@@ -441,6 +591,23 @@ typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9
[package.extras]
jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"]
+[[package]]
+name = "s3fs"
+version = "2022.5.0"
+description = "Convenient Filesystem interface over S3"
+category = "main"
+optional = true
+python-versions = ">= 3.7"
+
+[package.dependencies]
+aiobotocore = ">=2.3.0,<2.4.0"
+aiohttp = "<=4"
+fsspec = "2022.5.0"
+
+[package.extras]
+awscli = ["aiobotocore[awscli] (>=2.3.0,<2.4.0)"]
+boto3 = ["aiobotocore[boto3] (>=2.3.0,<2.4.0)"]
+
[[package]]
name = "six"
version = "1.16.0"
@@ -519,6 +686,26 @@ platformdirs = ">=2.4,<3"
docs = ["proselint (>=0.13)", "sphinx (>=5.1.1)", "sphinx-argparse (>=0.3.1)", "sphinx-rtd-theme (>=1)", "towncrier (>=21.9)"]
testing = ["coverage (>=6.2)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=21.3)", "pytest (>=7.0.1)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.2)", "pytest-mock (>=3.6.1)", "pytest-randomly (>=3.10.3)", "pytest-timeout (>=2.1)"]
+[[package]]
+name = "wrapt"
+version = "1.14.1"
+description = "Module for decorators, wrappers and monkey patching."
+category = "main"
+optional = true
+python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7"
+
+[[package]]
+name = "yarl"
+version = "1.8.1"
+description = "Yet another URL library"
+category = "main"
+optional = true
+python-versions = ">=3.7"
+
+[package.dependencies]
+idna = ">=2.0"
+multidict = ">=4.0"
+
[[package]]
name = "zipp"
version = "3.8.1"
@@ -549,16 +736,23 @@ cffi = ["cffi (>=1.11)"]
hive = ["thrift"]
pyarrow = ["pyarrow"]
python-snappy = ["zstandard"]
+s3fs = ["s3fs"]
snappy = ["python-snappy"]
[metadata]
lock-version = "1.1"
python-versions = "^3.8"
-content-hash = "279aca560c214b2c2f4583a9c13d586b1e58c29d121f8b97798efe9e0612ab98"
+content-hash = "7c6acb4d397c3920d6d6a47ce29593e9cb4799cf8e676ddb4d8dea2d2e9e43e5"
[metadata.files]
+aiobotocore = []
+aiohttp = []
+aioitertools = []
+aiosignal = []
+async-timeout = []
atomicwrites = []
attrs = []
+botocore = []
certifi = [
{file = "certifi-2022.6.15-py3-none-any.whl", hash = "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412"},
{file = "certifi-2022.6.15.tar.gz", hash = "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d"},
@@ -651,6 +845,8 @@ docutils = [
]
fastavro = []
filelock = []
+frozenlist = []
+fsspec = []
identify = []
idna = [
{file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"},
@@ -664,6 +860,7 @@ iniconfig = [
{file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"},
{file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"},
]
+jmespath = []
mmh3 = [
{file = "mmh3-3.0.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:23912dde2ad4f701926948dd8e79a0e42b000f73962806f153931f52985e1e07"},
{file = "mmh3-3.0.0-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:07f1308a410dc406d6a3c282a685728d00a87f3ed684f012671b96d6cc6a41c3"},
@@ -692,6 +889,7 @@ mmh3 = [
{file = "mmh3-3.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:bd870aedd9189eff1cf4e1687869b56c7e9461ee869789139c3e704009e5c227"},
{file = "mmh3-3.0.0.tar.gz", hash = "sha256:d1ec578c09a07d3518ec9be540b87546397fa3455de73c166fcce51eaa5c41c5"},
]
+multidict = []
nodeenv = [
{file = "nodeenv-1.7.0-py2.py3-none-any.whl", hash = "sha256:27083a7b96a25f2f5e1d8cb4b6317ee8aeda3bdd121394e5ac54e498028a042e"},
{file = "nodeenv-1.7.0.tar.gz", hash = "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b"},
@@ -721,42 +919,42 @@ pycparser = [
{file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"},
]
pydantic = [
- {file = "pydantic-1.10.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7e34e46dd08dafd4c75b8378efe3eae7d8e5212950fcd894d86c1df2dcfb80fe"},
- {file = "pydantic-1.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4af55f33ae5be6cccecd4fa462630daffef1f161f60c3f194b24eca705d50748"},
- {file = "pydantic-1.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1856bc6640aced42886f7ee48f5ed1fa5adf35e34064b5f9532b52d5a3b8a0d3"},
- {file = "pydantic-1.10.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d73ae7e210929a1b7d288034835dd787e5b0597192d58ab7342bacbeec0f33df"},
- {file = "pydantic-1.10.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a1192c17667d21652ab93b5eecd1a776cd0a4e384ea8c331bb830c9d130293af"},
- {file = "pydantic-1.10.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:026427be4e251f876e7519a63af37ae5ebb8b593ca8b02180bdc6becd1ea4ef4"},
- {file = "pydantic-1.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:d1dffae1f219d06a997ec78d1d2daafdbfecf243ad8eb36bfbcbc73e30e17385"},
- {file = "pydantic-1.10.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b549eebe8de4e50fc3b4f8c1f9cc2f731d91787fc3f7d031561668377b8679bc"},
- {file = "pydantic-1.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a0ba8710bfdaddb7424c05ad2dc1da04796003751eac6ad30c218ac1d68a174e"},
- {file = "pydantic-1.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f0985ba95af937389c9ce8d747138417303569cb736bd12469646ef53cd66e1c"},
- {file = "pydantic-1.10.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d484fbbe6267b6c936a6d005d5170ab553f3f4367348c7e88d3e17f0a7179981"},
- {file = "pydantic-1.10.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:9500586151cd56a20bacb8f1082df1b4489000120d1c7ddc44c8b20870e8adbd"},
- {file = "pydantic-1.10.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1b5212604aaf5954e9a7cea8f0c60d6dbef996aa7b41edefd329e6b5011ce8cf"},
- {file = "pydantic-1.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:39212b3853eea165a3cda11075d5b7d09d4291fcbc3c0ecefd23797ee21b29e9"},
- {file = "pydantic-1.10.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b3e3aed33fbd9518cf508d5415a58af683743d53dc5e58953973d73605774f34"},
- {file = "pydantic-1.10.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ed4e5c18cac70fadd4cf339f444c4f1795f0876dfd5b70cf0a841890b52f0001"},
- {file = "pydantic-1.10.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:45a6d0a9fdaad2a27ea69aec4659705ed8f60a5664e892c73e2b977d8f5166cc"},
- {file = "pydantic-1.10.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:158f1479367da20914961b5406ac3b29dfe1d858ae2af96c444f73543defcf0c"},
- {file = "pydantic-1.10.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:172aaeeaff8fc3ac326fb8a2934a063ca0938586c5fe8848285052de83a240f7"},
- {file = "pydantic-1.10.0-cp37-cp37m-win_amd64.whl", hash = "sha256:231b19c010288bfbfdcd3f79df38b5ff893c6547cd8c7d006203435790b22815"},
- {file = "pydantic-1.10.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:22206c152f9b86c0ee169928f9c24e1c0c566edb2462600b298ccb04860961aa"},
- {file = "pydantic-1.10.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d8ef840ef803ef17a7bd52480eb85faca0eed728d70233fd560f7d1066330247"},
- {file = "pydantic-1.10.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f99b4de6936a0f9fe255d1c7fdc447700ddd027c9ad38a612d453ed5fc7d6d0"},
- {file = "pydantic-1.10.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:134b4fd805737496ce4efd24ce2f8da0e08c66dcfc054fee1a19673eec780f2c"},
- {file = "pydantic-1.10.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:c4c76af6ad47bc46cf16bd0e4a5e536a7a2bec0dec14ea08b712daa6645bf293"},
- {file = "pydantic-1.10.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e03402b0a6b23a2d0b9ee31e45d80612c95562b5af8b5c900171b9d9015ddc5f"},
- {file = "pydantic-1.10.0-cp38-cp38-win_amd64.whl", hash = "sha256:3a3a60fcb5ce08cab593b7978d02db67b8d153e9d582adab7c0b69d7200d78be"},
- {file = "pydantic-1.10.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d8e5c5a50821c55b76dcf422610225cb7e44685cdd81832d0d504fa8c9343f35"},
- {file = "pydantic-1.10.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:645b83297a9428a675c98c1f69a7237a381900e34f23245c0ea73d74e454bf68"},
- {file = "pydantic-1.10.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95ab3f31f35dc4f8fc85b04d13569e5fdc9de2d3050ae64c1fdc3430dfe7d92d"},
- {file = "pydantic-1.10.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e290915a0ed53d3c59d6071fc7d2c843ed04c33affcd752dd1f3daa859b44a76"},
- {file = "pydantic-1.10.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:af669da39ede365069dbc5de56564b011e3353f801acdbdd7145002a78abc3d9"},
- {file = "pydantic-1.10.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8e796f915762dec4678fafc89b1f0441ab9209517a8a682ddb3f988f7ffe0827"},
- {file = "pydantic-1.10.0-cp39-cp39-win_amd64.whl", hash = "sha256:652727f9e1d3ae30bd8a4dfbebcafd50df45277b97f3deabbbfedcf731f94aa5"},
- {file = "pydantic-1.10.0-py3-none-any.whl", hash = "sha256:4d2b9258f5bd2d129bd4cf2d31f9d40094b9ed6ef64896e2f7a70729b2d599ea"},
- {file = "pydantic-1.10.0.tar.gz", hash = "sha256:e13788fcad1baf5eb3236856b2a9a74f7dac6b3ea7ca1f60a4ad8bad4239cf4c"},
+ {file = "pydantic-1.10.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:221166d99726238f71adc4fa9f3e94063a10787574b966f86a774559e709ac5a"},
+ {file = "pydantic-1.10.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a90e85d95fd968cd7cae122e0d3e0e1f6613bc88c1ff3fe838ac9785ea4b1c4c"},
+ {file = "pydantic-1.10.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2157aaf5718c648eaec9e654a34179ae42ffc363dc3ad058538a4f3ecbd9341"},
+ {file = "pydantic-1.10.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6142246fc9adb51cadaeb84fb52a86f3adad4c6a7b0938a5dd0b1356b0088217"},
+ {file = "pydantic-1.10.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:60dad97a09b6f44690c05467a4f397b62bfc2c839ac39102819d6979abc2be0d"},
+ {file = "pydantic-1.10.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d6f5bcb59d33ec46621dae76e714c53035087666cac80c81c9047a84f3ff93d0"},
+ {file = "pydantic-1.10.1-cp310-cp310-win_amd64.whl", hash = "sha256:522906820cd60e63c7960ba83078bf2d2ad2dd0870bf68248039bcb1ec3eb0a4"},
+ {file = "pydantic-1.10.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d545c89d88bdd5559db17aeb5a61a26799903e4bd76114779b3bf1456690f6ce"},
+ {file = "pydantic-1.10.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ad2374b5b3b771dcc6e2f6e0d56632ab63b90e9808b7a73ad865397fcdb4b2cd"},
+ {file = "pydantic-1.10.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90e02f61b7354ed330f294a437d0bffac9e21a5d46cb4cc3c89d220e497db7ac"},
+ {file = "pydantic-1.10.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cc5ffe7bd0b4778fa5b7a5f825c52d6cfea3ae2d9b52b05b9b1d97e36dee23a8"},
+ {file = "pydantic-1.10.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7acb7b66ffd2bc046eaff0063df84c83fc3826722d5272adaeadf6252e17f691"},
+ {file = "pydantic-1.10.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7e6786ed5faa559dea5a77f6d2de9a08d18130de9344533535d945f34bdcd42e"},
+ {file = "pydantic-1.10.1-cp311-cp311-win_amd64.whl", hash = "sha256:c7bf8ff1d18186eb0cbe42bd9bfb4cbf7fde1fd01b8608925458990c21f202f0"},
+ {file = "pydantic-1.10.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:14a5babda137a294df7ad5f220986d79bbb87fdeb332c6ded61ce19da7f5f3bf"},
+ {file = "pydantic-1.10.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5659cb9c6b3d27fc0067025c4f5a205f5e838232a4a929b412781117c2343d44"},
+ {file = "pydantic-1.10.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c8d70fb91b03c32d2e857b071a22a5225e6b625ca82bd2cc8dd729d88e0bd200"},
+ {file = "pydantic-1.10.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:9a93be313e40f12c6f2cb84533b226bbe23d0774872e38d83415e6890215e3a6"},
+ {file = "pydantic-1.10.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d55aeb01bb7bd7c7e1bd904668a4a2ffcbb1c248e7ae9eb40a272fd7e67dd98b"},
+ {file = "pydantic-1.10.1-cp37-cp37m-win_amd64.whl", hash = "sha256:43d41b6f13706488e854729955ba8f740e6ec375cd16b72b81dc24b9d84f0d15"},
+ {file = "pydantic-1.10.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f31ffe0e38805a0e6410330f78147bb89193b136d7a5f79cae60d3e849b520a6"},
+ {file = "pydantic-1.10.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:8eee69eda7674977b079a21e7bf825b59d8bf15145300e8034ed3eb239ac444f"},
+ {file = "pydantic-1.10.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f927bff6c319fc92e0a2cbeb2609b5c1cd562862f4b54ec905e353282b7c8b1"},
+ {file = "pydantic-1.10.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb1bc3f8fef6ba36977108505e90558911e7fbccb4e930805d5dd90891b56ff4"},
+ {file = "pydantic-1.10.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:96ab6ce1346d14c6e581a69c333bdd1b492df9cf85ad31ad77a8aa42180b7e09"},
+ {file = "pydantic-1.10.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:444cf220a12134da1cd42fe4f45edff622139e10177ce3d8ef2b4f41db1291b2"},
+ {file = "pydantic-1.10.1-cp38-cp38-win_amd64.whl", hash = "sha256:dbfbff83565b4514dd8cebc8b8c81a12247e89427ff997ad0a9da7b2b1065c12"},
+ {file = "pydantic-1.10.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5327406f4bfd5aee784e7ad2a6a5fdd7171c19905bf34cb1994a1ba73a87c468"},
+ {file = "pydantic-1.10.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1072eae28bf034a311764c130784e8065201a90edbca10f495c906737b3bd642"},
+ {file = "pydantic-1.10.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce901335667a68dfbc10dd2ee6c0d676b89210d754441c2469fbc37baf7ee2ed"},
+ {file = "pydantic-1.10.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54d6465cd2112441305faf5143a491b40de07a203116b5755a2108e36b25308d"},
+ {file = "pydantic-1.10.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:2b5e5e7a0ec96704099e271911a1049321ba1afda92920df0769898a7e9a1298"},
+ {file = "pydantic-1.10.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ae43704358304da45c1c3dd7056f173c618b252f91594bcb6d6f6b4c6c284dee"},
+ {file = "pydantic-1.10.1-cp39-cp39-win_amd64.whl", hash = "sha256:2d7da49229ffb1049779a5a6c1c50a26da164bd053cf8ee9042197dc08a98259"},
+ {file = "pydantic-1.10.1-py3-none-any.whl", hash = "sha256:f8b10e59c035ff3dcc9791619d6e6c5141e0fa5cbe264e19e267b8d523b210bf"},
+ {file = "pydantic-1.10.1.tar.gz", hash = "sha256:d41bb80347a8a2d51fbd6f1748b42aca14541315878447ba159617544712f770"},
]
pygments = []
pyparsing = [
@@ -771,6 +969,7 @@ pytest-checkdocs = [
{file = "pytest-checkdocs-2.7.1.tar.gz", hash = "sha256:2b33b85eddfe5846a69bea4a759303e2d5a3be11d03bc7149f5ba1ef47e6c1ae"},
{file = "pytest_checkdocs-2.7.1-py3-none-any.whl", hash = "sha256:294898c64c9ce1a178edc6660e48da23c7543bfd5a1cea7f0ca4c167745d8461"},
]
+python-dateutil = []
python-snappy = [
{file = "python-snappy-0.6.1.tar.gz", hash = "sha256:b6a107ab06206acc5359d4c5632bd9b22d448702a79b3169b0c62e0fb808bb2a"},
{file = "python_snappy-0.6.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b7f920eaf46ebf41bd26f9df51c160d40f9e00b7b48471c3438cb8d027f7fb9b"},
@@ -865,6 +1064,7 @@ requests-mock = [
{file = "requests_mock-1.10.0-py2.py3-none-any.whl", hash = "sha256:2fdbb637ad17ee15c06f33d31169e71bf9fe2bdb7bc9da26185be0dd8d842699"},
]
rich = []
+s3fs = []
six = [
{file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"},
{file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"},
@@ -887,5 +1087,72 @@ virtualenv = [
{file = "virtualenv-20.16.4-py3-none-any.whl", hash = "sha256:035ed57acce4ac35c82c9d8802202b0e71adac011a511ff650cbcf9635006a22"},
{file = "virtualenv-20.16.4.tar.gz", hash = "sha256:014f766e4134d0008dcaa1f95bafa0fb0f575795d07cae50b1bee514185d6782"},
]
+wrapt = [
+ {file = "wrapt-1.14.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:1b376b3f4896e7930f1f772ac4b064ac12598d1c38d04907e696cc4d794b43d3"},
+ {file = "wrapt-1.14.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:903500616422a40a98a5a3c4ff4ed9d0066f3b4c951fa286018ecdf0750194ef"},
+ {file = "wrapt-1.14.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:5a9a0d155deafd9448baff28c08e150d9b24ff010e899311ddd63c45c2445e28"},
+ {file = "wrapt-1.14.1-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:ddaea91abf8b0d13443f6dac52e89051a5063c7d014710dcb4d4abb2ff811a59"},
+ {file = "wrapt-1.14.1-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:36f582d0c6bc99d5f39cd3ac2a9062e57f3cf606ade29a0a0d6b323462f4dd87"},
+ {file = "wrapt-1.14.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:7ef58fb89674095bfc57c4069e95d7a31cfdc0939e2a579882ac7d55aadfd2a1"},
+ {file = "wrapt-1.14.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:e2f83e18fe2f4c9e7db597e988f72712c0c3676d337d8b101f6758107c42425b"},
+ {file = "wrapt-1.14.1-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:ee2b1b1769f6707a8a445162ea16dddf74285c3964f605877a20e38545c3c462"},
+ {file = "wrapt-1.14.1-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:833b58d5d0b7e5b9832869f039203389ac7cbf01765639c7309fd50ef619e0b1"},
+ {file = "wrapt-1.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:80bb5c256f1415f747011dc3604b59bc1f91c6e7150bd7db03b19170ee06b320"},
+ {file = "wrapt-1.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:07f7a7d0f388028b2df1d916e94bbb40624c59b48ecc6cbc232546706fac74c2"},
+ {file = "wrapt-1.14.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:02b41b633c6261feff8ddd8d11c711df6842aba629fdd3da10249a53211a72c4"},
+ {file = "wrapt-1.14.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2fe803deacd09a233e4762a1adcea5db5d31e6be577a43352936179d14d90069"},
+ {file = "wrapt-1.14.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:257fd78c513e0fb5cdbe058c27a0624c9884e735bbd131935fd49e9fe719d310"},
+ {file = "wrapt-1.14.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:4fcc4649dc762cddacd193e6b55bc02edca674067f5f98166d7713b193932b7f"},
+ {file = "wrapt-1.14.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:11871514607b15cfeb87c547a49bca19fde402f32e2b1c24a632506c0a756656"},
+ {file = "wrapt-1.14.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8ad85f7f4e20964db4daadcab70b47ab05c7c1cf2a7c1e51087bfaa83831854c"},
+ {file = "wrapt-1.14.1-cp310-cp310-win32.whl", hash = "sha256:a9a52172be0b5aae932bef82a79ec0a0ce87288c7d132946d645eba03f0ad8a8"},
+ {file = "wrapt-1.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:6d323e1554b3d22cfc03cd3243b5bb815a51f5249fdcbb86fda4bf62bab9e164"},
+ {file = "wrapt-1.14.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:43ca3bbbe97af00f49efb06e352eae40434ca9d915906f77def219b88e85d907"},
+ {file = "wrapt-1.14.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:6b1a564e6cb69922c7fe3a678b9f9a3c54e72b469875aa8018f18b4d1dd1adf3"},
+ {file = "wrapt-1.14.1-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:00b6d4ea20a906c0ca56d84f93065b398ab74b927a7a3dbd470f6fc503f95dc3"},
+ {file = "wrapt-1.14.1-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:a85d2b46be66a71bedde836d9e41859879cc54a2a04fad1191eb50c2066f6e9d"},
+ {file = "wrapt-1.14.1-cp35-cp35m-win32.whl", hash = "sha256:dbcda74c67263139358f4d188ae5faae95c30929281bc6866d00573783c422b7"},
+ {file = "wrapt-1.14.1-cp35-cp35m-win_amd64.whl", hash = "sha256:b21bb4c09ffabfa0e85e3a6b623e19b80e7acd709b9f91452b8297ace2a8ab00"},
+ {file = "wrapt-1.14.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:9e0fd32e0148dd5dea6af5fee42beb949098564cc23211a88d799e434255a1f4"},
+ {file = "wrapt-1.14.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9736af4641846491aedb3c3f56b9bc5568d92b0692303b5a305301a95dfd38b1"},
+ {file = "wrapt-1.14.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5b02d65b9ccf0ef6c34cba6cf5bf2aab1bb2f49c6090bafeecc9cd81ad4ea1c1"},
+ {file = "wrapt-1.14.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21ac0156c4b089b330b7666db40feee30a5d52634cc4560e1905d6529a3897ff"},
+ {file = "wrapt-1.14.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:9f3e6f9e05148ff90002b884fbc2a86bd303ae847e472f44ecc06c2cd2fcdb2d"},
+ {file = "wrapt-1.14.1-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:6e743de5e9c3d1b7185870f480587b75b1cb604832e380d64f9504a0535912d1"},
+ {file = "wrapt-1.14.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:d79d7d5dc8a32b7093e81e97dad755127ff77bcc899e845f41bf71747af0c569"},
+ {file = "wrapt-1.14.1-cp36-cp36m-win32.whl", hash = "sha256:81b19725065dcb43df02b37e03278c011a09e49757287dca60c5aecdd5a0b8ed"},
+ {file = "wrapt-1.14.1-cp36-cp36m-win_amd64.whl", hash = "sha256:b014c23646a467558be7da3d6b9fa409b2c567d2110599b7cf9a0c5992b3b471"},
+ {file = "wrapt-1.14.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:88bd7b6bd70a5b6803c1abf6bca012f7ed963e58c68d76ee20b9d751c74a3248"},
+ {file = "wrapt-1.14.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b5901a312f4d14c59918c221323068fad0540e34324925c8475263841dbdfe68"},
+ {file = "wrapt-1.14.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d77c85fedff92cf788face9bfa3ebaa364448ebb1d765302e9af11bf449ca36d"},
+ {file = "wrapt-1.14.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d649d616e5c6a678b26d15ece345354f7c2286acd6db868e65fcc5ff7c24a77"},
+ {file = "wrapt-1.14.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7d2872609603cb35ca513d7404a94d6d608fc13211563571117046c9d2bcc3d7"},
+ {file = "wrapt-1.14.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:ee6acae74a2b91865910eef5e7de37dc6895ad96fa23603d1d27ea69df545015"},
+ {file = "wrapt-1.14.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:2b39d38039a1fdad98c87279b48bc5dce2c0ca0d73483b12cb72aa9609278e8a"},
+ {file = "wrapt-1.14.1-cp37-cp37m-win32.whl", hash = "sha256:60db23fa423575eeb65ea430cee741acb7c26a1365d103f7b0f6ec412b893853"},
+ {file = "wrapt-1.14.1-cp37-cp37m-win_amd64.whl", hash = "sha256:709fe01086a55cf79d20f741f39325018f4df051ef39fe921b1ebe780a66184c"},
+ {file = "wrapt-1.14.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8c0ce1e99116d5ab21355d8ebe53d9460366704ea38ae4d9f6933188f327b456"},
+ {file = "wrapt-1.14.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e3fb1677c720409d5f671e39bac6c9e0e422584e5f518bfd50aa4cbbea02433f"},
+ {file = "wrapt-1.14.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:642c2e7a804fcf18c222e1060df25fc210b9c58db7c91416fb055897fc27e8cc"},
+ {file = "wrapt-1.14.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b7c050ae976e286906dd3f26009e117eb000fb2cf3533398c5ad9ccc86867b1"},
+ {file = "wrapt-1.14.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef3f72c9666bba2bab70d2a8b79f2c6d2c1a42a7f7e2b0ec83bb2f9e383950af"},
+ {file = "wrapt-1.14.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:01c205616a89d09827986bc4e859bcabd64f5a0662a7fe95e0d359424e0e071b"},
+ {file = "wrapt-1.14.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:5a0f54ce2c092aaf439813735584b9537cad479575a09892b8352fea5e988dc0"},
+ {file = "wrapt-1.14.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2cf71233a0ed05ccdabe209c606fe0bac7379fdcf687f39b944420d2a09fdb57"},
+ {file = "wrapt-1.14.1-cp38-cp38-win32.whl", hash = "sha256:aa31fdcc33fef9eb2552cbcbfee7773d5a6792c137b359e82879c101e98584c5"},
+ {file = "wrapt-1.14.1-cp38-cp38-win_amd64.whl", hash = "sha256:d1967f46ea8f2db647c786e78d8cc7e4313dbd1b0aca360592d8027b8508e24d"},
+ {file = "wrapt-1.14.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3232822c7d98d23895ccc443bbdf57c7412c5a65996c30442ebe6ed3df335383"},
+ {file = "wrapt-1.14.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:988635d122aaf2bdcef9e795435662bcd65b02f4f4c1ae37fbee7401c440b3a7"},
+ {file = "wrapt-1.14.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cca3c2cdadb362116235fdbd411735de4328c61425b0aa9f872fd76d02c4e86"},
+ {file = "wrapt-1.14.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d52a25136894c63de15a35bc0bdc5adb4b0e173b9c0d07a2be9d3ca64a332735"},
+ {file = "wrapt-1.14.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40e7bc81c9e2b2734ea4bc1aceb8a8f0ceaac7c5299bc5d69e37c44d9081d43b"},
+ {file = "wrapt-1.14.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b9b7a708dd92306328117d8c4b62e2194d00c365f18eff11a9b53c6f923b01e3"},
+ {file = "wrapt-1.14.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:6a9a25751acb379b466ff6be78a315e2b439d4c94c1e99cb7266d40a537995d3"},
+ {file = "wrapt-1.14.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:34aa51c45f28ba7f12accd624225e2b1e5a3a45206aa191f6f9aac931d9d56fe"},
+ {file = "wrapt-1.14.1-cp39-cp39-win32.whl", hash = "sha256:dee0ce50c6a2dd9056c20db781e9c1cfd33e77d2d569f5d1d9321c641bb903d5"},
+ {file = "wrapt-1.14.1-cp39-cp39-win_amd64.whl", hash = "sha256:dee60e1de1898bde3b238f18340eec6148986da0455d8ba7848d50470a7a32fb"},
+ {file = "wrapt-1.14.1.tar.gz", hash = "sha256:380a85cf89e0e69b7cfbe2ea9f765f004ff419f34194018a6827ac0e3edfed4d"},
+]
+yarl = []
zipp = []
zstandard = []
diff --git a/python/pyiceberg/io/fsspec.py b/python/pyiceberg/io/fsspec.py
new file mode 100644
index 0000000000..769c309a84
--- /dev/null
+++ b/python/pyiceberg/io/fsspec.py
@@ -0,0 +1,196 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""FileIO implementation for reading and writing table files that uses fsspec compatible filesystems"""
+
+from functools import lru_cache
+from typing import Callable, Union
+from urllib.parse import urlparse
+
+from fsspec import AbstractFileSystem
+
+from pyiceberg.io import FileIO, InputFile, OutputFile
+from pyiceberg.typedef import Properties
+
+
+def _s3(properties: Properties, **fs_properties) -> AbstractFileSystem:
+ from s3fs import S3FileSystem
+
+ client_kwargs = {
+ "endpoint_url": properties.get("s3.endpoint"),
+ "aws_access_key_id": properties.get("s3.access-key-id"),
+ "aws_secret_access_key": properties.get("s3.secret-access-key"),
+ }
+
+ config_kwargs = {"signature_version": properties.get("s3.signer")}
+
+ return S3FileSystem(client_kwargs=client_kwargs, config_kwargs=config_kwargs, **fs_properties)
+
+
+SCHEME_TO_FS = {
+ "s3": _s3,
+ "s3a": _s3,
+ "s3n": _s3,
+}
+
+
+class FsspecInputFile(InputFile):
+ """An input file implementation for the FsspecFileIO
+
+ Args:
+ location(str): A URI to a file location
+ fs(AbstractFileSystem): An fsspec filesystem instance
+ """
+
+ def __init__(self, location: str, fs: AbstractFileSystem):
+ self._fs = fs
+ super().__init__(location=location)
+
+ def __len__(self) -> int:
+ """Returns the total length of the file, in bytes"""
+ object_info = self._fs.info(self.location)
+ if size := object_info.get("Size"):
+ return size
+ elif size := object_info.get("size"):
+ return size
+ raise RuntimeError(f"Cannot retrieve object info: {self.location}")
+
+ def exists(self) -> bool:
+ """Checks whether the location exists"""
+ return self._fs.lexists(self.location)
+
+ def open(self):
+ """Create an input stream for reading the contents of the file
+
+ Returns:
+ OpenFile: An fsspec compliant file-like object
+ """
+ return self._fs.open(self.location, "rb")
+
+
+class FsspecOutputFile(OutputFile):
+ """An output file implementation for the FsspecFileIO
+
+ Args:
+ location(str): A URI to a file location
+ fs(AbstractFileSystem): An fsspec filesystem instance
+ """
+
+ def __init__(self, location: str, fs: AbstractFileSystem):
+ self._fs = fs
+ super().__init__(location=location)
+
+ def __len__(self) -> int:
+ """Returns the total length of the file, in bytes"""
+ object_info = self._fs.info(self.location)
+ if size := object_info.get("Size"):
+ return size
+ elif size := object_info.get("size"):
+ return size
+ raise RuntimeError(f"Cannot retrieve object info: {self.location}")
+
+ def exists(self) -> bool:
+ """Checks whether the location exists"""
+ return self._fs.lexists(self.location)
+
+ def create(self, overwrite: bool = False):
+ """Create an output stream for reading the contents of the file
+
+ Args:
+ overwrite(bool): Whether to overwrite the file if it already exists
+
+ Returns:
+ OpenFile: An fsspec compliant file-like object
+
+ Raises:
+ FileExistsError: If the file already exists at the location and overwrite is set to False
+
+ Note:
+ If overwrite is set to False, a check is first performed to verify that the file does not exist.
+ This is not thread-safe and a possibility does exist that the file can be created by a concurrent
+ process after the existence check yet before the output stream is created. In such a case, the default
+ behavior will truncate the contents of the existing file when opening the output stream.
+ """
+ if not overwrite and self.exists():
+ raise FileExistsError(f"Cannot create file, file already exists: {self.location}")
+ return self._fs.open(self.location, "wb")
+
+ def to_input_file(self) -> FsspecInputFile:
+ """Returns a new FsspecInputFile for the location at `self.location`"""
+ return FsspecInputFile(location=self.location, fs=self._fs)
+
+
+class FsspecFileIO(FileIO):
+ """A FileIO implementation that uses fsspec"""
+
+ def __init__(self, properties: Properties):
+ self._scheme_to_fs = {}
+ self._scheme_to_fs.update(SCHEME_TO_FS)
+ self.get_fs: Callable = lru_cache(self._get_fs)
+ super().__init__(properties=properties)
+
+ def new_input(self, location: str) -> FsspecInputFile:
+ """Get an FsspecInputFile instance to read bytes from the file at the given location
+
+ Args:
+ location(str): A URI or a path to a local file
+
+ Returns:
+ FsspecInputFile: An FsspecInputFile instance for the given location
+ """
+ uri = urlparse(location)
+ fs = self.get_fs(uri.scheme)
+ return FsspecInputFile(location=location, fs=fs)
+
+ def new_output(self, location: str) -> FsspecOutputFile:
+ """Get an FsspecOutputFile instance to write bytes to the file at the given location
+
+ Args:
+ location(str): A URI or a path to a local file
+
+ Returns:
+ FsspecOutputFile: An FsspecOutputFile instance for the given location
+ """
+ uri = urlparse(location)
+ fs = self.get_fs(uri.scheme)
+ return FsspecOutputFile(location=location, fs=fs)
+
+ def delete(self, location: Union[str, InputFile, OutputFile]) -> None:
+ """Delete the file at the given location
+
+ Args:
+ location(str, InputFile, OutputFile): The URI to the file--if an InputFile instance or an
+ OutputFile instance is provided, the location attribute for that instance is used as the location
+ to delete
+ """
+ if isinstance(location, (InputFile, OutputFile)):
+ str_location = location.location # Use InputFile or OutputFile location
+ else:
+ str_location = location
+
+ uri = urlparse(str_location)
+ fs = self.get_fs(uri.scheme)
+ fs.rm(str_location)
+
+ def _get_fs(self, scheme: str) -> AbstractFileSystem:
+ """Get a filesystem for a specific scheme"""
+ if scheme not in self._scheme_to_fs:
+ raise ValueError(f"No registered filesystem for scheme: {scheme}")
+ return self._scheme_to_fs[scheme](self.properties, **self._fs_properties())
+
+ def _fs_properties(self):
+ """Get fs properties from the file-io property map"""
+ return {k[3:]: v for k, v in self.properties.items() if k.startswith("fs_")}
diff --git a/python/pyproject.toml b/python/pyproject.toml
index b55a6ce5c0..3d6d9d25df 100644
--- a/python/pyproject.toml
+++ b/python/pyproject.toml
@@ -48,6 +48,7 @@ click = "^8.1.3"
rich = "^12.5.1"
pydantic = "^1.9.2"
+fsspec = "2022.5.0"
pyarrow = { version = "^9.0.0", optional = true }
@@ -57,6 +58,8 @@ python-snappy = { version = "^0.6.1", optional = true }
thrift = { version = "^0.16.0", optional = true }
+s3fs = { version = "2022.5.0", optional = true}
+
[tool.poetry.dev-dependencies]
pytest = "^7.0.0"
pytest-checkdocs = "^2.0.0"
@@ -79,6 +82,12 @@ pyarrow = ["pyarrow"]
snappy = ["python-snappy"]
python-snappy = ["zstandard"]
hive = ["thrift"]
+s3fs = ["s3fs"]
+
+[tool.pytest.ini_options]
+markers = [
+ "s3: marks a test as requiring access to s3 compliant storage (use with --aws-access-key-id, ----aws-secret-access-key, and --endpoint-url args)"
+]
[tool.black]
line-length = 130
@@ -148,5 +157,13 @@ ignore_missing_imports = true
module = "rich.*"
ignore_missing_imports = true
+[[tool.mypy.overrides]]
+module = "fsspec.*"
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "s3fs.*"
+ignore_missing_imports = true
+
[tool.coverage.run]
source = ['pyiceberg/']
diff --git a/python/tests/conftest.py b/python/tests/conftest.py
index b4d2e005fe..4030fbe9f1 100644
--- a/python/tests/conftest.py
+++ b/python/tests/conftest.py
@@ -37,6 +37,7 @@ from pyiceberg.io import (
InputFile,
OutputFile,
OutputStream,
+ fsspec,
)
from pyiceberg.schema import Schema
from pyiceberg.types import (
@@ -56,6 +57,16 @@ from tests.catalog.test_base import InMemoryCatalog
from tests.io.test_io import LocalInputFile
+def pytest_addoption(parser):
+ parser.addoption(
+ "--s3.endpoint", action="store", default="http://localhost:9000", help="The S3 endpoint URL for tests marked as s3"
+ )
+ parser.addoption("--s3.access-key-id", action="store", default="admin", help="The AWS access key ID for tests marked as s3")
+ parser.addoption(
+ "--s3.secret-access-key", action="store", default="password", help="The AWS secret access key ID for tests marked as s3"
+ )
+
+
class FooStruct:
"""An example of an object that abides by StructProtocol"""
@@ -1111,3 +1122,13 @@ def iceberg_manifest_entry_schema() -> Schema:
schema_id=1,
identifier_field_ids=[],
)
+
+
+@pytest.fixture
+def fsspec_fileio(request):
+ properties = {
+ "s3.endpoint": request.config.getoption("--s3.endpoint"),
+ "s3.access-key-id": request.config.getoption("--s3.access-key-id"),
+ "s3.secret-access-key": request.config.getoption("--s3.secret-access-key"),
+ }
+ return fsspec.FsspecFileIO(properties=properties)
diff --git a/python/tests/io/test_fsspec.py b/python/tests/io/test_fsspec.py
new file mode 100644
index 0000000000..2c05740fc3
--- /dev/null
+++ b/python/tests/io/test_fsspec.py
@@ -0,0 +1,200 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import uuid
+
+import pytest
+
+from pyiceberg.io import fsspec
+from tests.io.test_io import LocalInputFile
+
+
+@pytest.mark.s3
+def test_fsspec_new_input_file(fsspec_fileio):
+ """Test creating a new input file from an fsspec file-io"""
+ filename = str(uuid.uuid4())
+
+ input_file = fsspec_fileio.new_input(f"s3://warehouse/{filename}")
+
+ assert isinstance(input_file, fsspec.FsspecInputFile)
+ assert input_file.location == f"s3://warehouse/{filename}"
+
+
+@pytest.mark.s3
+def test_fsspec_new_s3_output_file(fsspec_fileio):
+ """Test creating a new output file from an fsspec file-io"""
+ filename = str(uuid.uuid4())
+
+ output_file = fsspec_fileio.new_output(f"s3://warehouse/{filename}")
+
+ assert isinstance(output_file, fsspec.FsspecOutputFile)
+ assert output_file.location == f"s3://warehouse/{filename}"
+
+
+@pytest.mark.s3
+def test_fsspec_write_and_read_file(fsspec_fileio):
+ """Test writing and reading a file using FsspecInputFile and FsspecOutputFile"""
+ filename = str(uuid.uuid4())
+ output_file = fsspec_fileio.new_output(location=f"s3://warehouse/{filename}")
+ with output_file.create() as f:
+ f.write(b"foo")
+
+ input_file = fsspec_fileio.new_input(f"s3://warehouse/{filename}")
+ assert input_file.open().read() == b"foo"
+
+ fsspec_fileio.delete(input_file)
+
+
+@pytest.mark.s3
+def test_fsspec_getting_length_of_file(fsspec_fileio):
+ """Test getting the length of an FsspecInputFile and FsspecOutputFile"""
+ filename = str(uuid.uuid4())
+
+ output_file = fsspec_fileio.new_output(location=f"s3://warehouse/{filename}")
+ with output_file.create() as f:
+ f.write(b"foobar")
+
+ assert len(output_file) == 6
+
+ input_file = fsspec_fileio.new_input(location=f"s3://warehouse/{filename}")
+ assert len(input_file) == 6
+
+ fsspec_fileio.delete(output_file)
+
+
+@pytest.mark.s3
+def test_fsspec_file_tell(fsspec_fileio):
+ """Test finding cursor position for an fsspec file-io file"""
+
+ filename = str(uuid.uuid4())
+
+ output_file = fsspec_fileio.new_output(location=f"s3://warehouse/{filename}")
+ with output_file.create() as f:
+ f.write(b"foobar")
+
+ input_file = fsspec_fileio.new_input(location=f"s3://warehouse/{filename}")
+ f = input_file.open()
+
+ f.seek(0)
+ assert f.tell() == 0
+ f.seek(1)
+ assert f.tell() == 1
+ f.seek(3)
+ assert f.tell() == 3
+ f.seek(0)
+ assert f.tell() == 0
+
+
+@pytest.mark.s3
+def test_fsspec_read_specified_bytes_for_file(fsspec_fileio):
+ """Test reading a specified number of bytes from an fsspec file-io file"""
+
+ filename = str(uuid.uuid4())
+ output_file = fsspec_fileio.new_output(location=f"s3://warehouse/{filename}")
+ with output_file.create() as f:
+ f.write(b"foo")
+
+ input_file = fsspec_fileio.new_input(location=f"s3://warehouse/{filename}")
+ f = input_file.open()
+
+ f.seek(0)
+ assert b"f" == f.read(1)
+ f.seek(0)
+ assert b"fo" == f.read(2)
+ f.seek(1)
+ assert b"o" == f.read(1)
+ f.seek(1)
+ assert b"oo" == f.read(2)
+ f.seek(0)
+ assert b"foo" == f.read(999) # test reading amount larger than entire content length
+
+ fsspec_fileio.delete(input_file)
+
+
+@pytest.mark.s3
+def test_fsspec_raise_on_opening_file_not_found(fsspec_fileio):
+ """Test that an fsppec input file raises appropriately when the s3 file is not found"""
+
+ filename = str(uuid.uuid4())
+ input_file = fsspec_fileio.new_input(location=f"s3://warehouse/{filename}")
+ with pytest.raises(FileNotFoundError) as exc_info:
+ input_file.open().read()
+
+ assert filename in str(exc_info.value)
+
+
+@pytest.mark.s3
+def test_checking_if_a_file_exists(fsspec_fileio):
+ """Test checking if a file exists"""
+
+ non_existent_file = fsspec_fileio.new_input(location="s3://warehouse/does-not-exist.txt")
+ assert not non_existent_file.exists()
+
+ filename = str(uuid.uuid4())
+ output_file = fsspec_fileio.new_output(location=f"s3://warehouse/{filename}")
+ assert not output_file.exists()
+ with output_file.create() as f:
+ f.write(b"foo")
+
+ existing_input_file = fsspec_fileio.new_input(location=f"s3://warehouse/{filename}")
+ assert existing_input_file.exists()
+
+ existing_output_file = fsspec_fileio.new_output(location=f"s3://warehouse/{filename}")
+ assert existing_output_file.exists()
+
+ fsspec_fileio.delete(existing_output_file)
+
+
+@pytest.mark.s3
+def test_closing_a_file(fsspec_fileio):
+ """Test closing an output file and input file"""
+ filename = str(uuid.uuid4())
+ output_file = fsspec_fileio.new_output(location=f"s3://warehouse/{filename}")
+ with output_file.create() as f:
+ f.write(b"foo")
+ assert not f.closed
+ assert f.closed
+
+ input_file = fsspec_fileio.new_input(location=f"s3://warehouse/{filename}")
+ f = input_file.open()
+ assert not f.closed
+ f.close()
+ assert f.closed
+
+ fsspec_fileio.delete(f"s3://warehouse/{filename}")
+
+
+@pytest.mark.s3
+def test_fsspec_converting_an_outputfile_to_an_inputfile(fsspec_fileio):
+ """Test converting an output file to an input file"""
+ filename = str(uuid.uuid4())
+ output_file = fsspec_fileio.new_output(location=f"s3://warehouse/{filename}")
+ input_file = output_file.to_input_file()
+ assert input_file.location == output_file.location
+
+
+@pytest.mark.s3
+def test_writing_avro_file(generated_manifest_entry_file, fsspec_fileio):
+ """Test that bytes match when reading a local avro file, writing it using fsspec file-io, and then reading it again"""
+ filename = str(uuid.uuid4())
+ with LocalInputFile(generated_manifest_entry_file).open() as f:
+ b1 = f.read()
+ with fsspec_fileio.new_output(location=f"s3://warehouse/{filename}").create() as out_f:
+ out_f.write(b1)
+ with fsspec_fileio.new_input(location=f"s3://warehouse/{filename}").open() as in_f:
+ b2 = in_f.read()
+ assert b1 == b2 # Check that bytes of read from local avro file match bytes written to s3