You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by am...@apache.org on 2022/06/23 21:38:32 UTC
[arrow] branch master updated: ARROW-16092: [Python] Address docstrings in Filesystems (Implementations) (#13416)
This is an automated email from the ASF dual-hosted git repository.
amolina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new e93acea84c ARROW-16092: [Python] Address docstrings in Filesystems (Implementations) (#13416)
e93acea84c is described below
commit e93acea84c362f9299ba20471ba4e583465ab60d
Author: Alenka Frim <Al...@users.noreply.github.com>
AuthorDate: Thu Jun 23 23:38:24 2022 +0200
ARROW-16092: [Python] Address docstrings in Filesystems (Implementations) (#13416)
Authored-by: Alenka Frim <fr...@gmail.com>
Signed-off-by: Alessandro Molina <am...@turbogears.org>
---
python/pyarrow/_fs.pyx | 139 +++++++++++++++++++++++++++++++++++++++++++++++
python/pyarrow/_hdfs.pyx | 7 +++
python/pyarrow/_s3fs.pyx | 11 ++++
3 files changed, 157 insertions(+)
diff --git a/python/pyarrow/_fs.pyx b/python/pyarrow/_fs.pyx
index af5bebe7d6..5a98593868 100644
--- a/python/pyarrow/_fs.pyx
+++ b/python/pyarrow/_fs.pyx
@@ -782,6 +782,106 @@ cdef class LocalFileSystem(FileSystem):
use_mmap : bool, default False
Whether open_input_stream and open_input_file should return
a mmap'ed file or a regular file.
+
+ Examples
+ --------
+ Create a FileSystem object with LocalFileSystem constructor:
+
+ >>> from pyarrow import fs
+ >>> local = fs.LocalFileSystem()
+ >>> local
+ <pyarrow._fs.LocalFileSystem object at ...>
+
+ and write data on to the file:
+
+ >>> with local.open_output_stream('/tmp/local_fs.dat') as stream:
+ ... stream.write(b'data')
+ ...
+ 4
+ >>> with local.open_input_stream('/tmp/local_fs.dat') as stream:
+ ... print(stream.readall())
+ ...
+ b'data'
+
+ Create a FileSystem object inferred from a URI of the saved file:
+
+ >>> local_new, path = fs.LocalFileSystem().from_uri('/tmp/local_fs.dat')
+ >>> local_new
+ <pyarrow._fs.LocalFileSystem object at ...
+ >>> path
+ '/tmp/local_fs.dat'
+
+ Check if FileSystems `local` and `local_new` are equal:
+
+ >>> local.equals(local_new)
+ True
+
+ Compare two different FileSystems:
+
+ >>> local2 = fs.LocalFileSystem(use_mmap=True)
+ >>> local.equals(local2)
+ False
+
+ Copy a file and print out the data:
+
+ >>> local.copy_file('/tmp/local_fs.dat', '/tmp/local_fs-copy.dat')
+ >>> with local.open_input_stream('/tmp/local_fs-copy.dat') as stream:
+ ... print(stream.readall())
+ ...
+ b'data'
+
+ Open an output stream for appending, add text and print the new data:
+
+ >>> with local.open_append_stream('/tmp/local_fs-copy.dat') as f:
+ ... f.write(b'+newly added')
+ ...
+ 12
+ >>> with local.open_input_stream('/tmp/local_fs-copy.dat') as f:
+ ... print(f.readall())
+ ...
+ b'data+newly added'
+
+ Create a directory, copy a file into it and then delete the whole directory:
+
+ >>> local.create_dir('/tmp/new_folder')
+ >>> local.copy_file('/tmp/local_fs.dat', '/tmp/new_folder/local_fs.dat')
+ >>> local.get_file_info('/tmp/new_folder')
+ <FileInfo for '/tmp/new_folder': type=FileType.Directory>
+ >>> local.delete_dir('/tmp/new_folder')
+ >>> local.get_file_info('/tmp/new_folder')
+ <FileInfo for '/tmp/new_folder': type=FileType.NotFound>
+
+ Create a directory, copy a file into it and then delete
+ the content of the directory:
+
+ >>> local.create_dir('/tmp/new_folder')
+ >>> local.copy_file('/tmp/local_fs.dat', '/tmp/new_folder/local_fs.dat')
+ >>> local.get_file_info('/tmp/new_folder/local_fs.dat')
+ <FileInfo for '/tmp/new_folder/local_fs.dat': type=FileType.File, size=4>
+ >>> local.delete_dir_contents('/tmp/new_folder')
+ >>> local.get_file_info('/tmp/new_folder')
+ <FileInfo for '/tmp/new_folder': type=FileType.Directory>
+ >>> local.get_file_info('/tmp/new_folder/local_fs.dat')
+ <FileInfo for '/tmp/new_folder/local_fs.dat': type=FileType.NotFound>
+
+ Create a directory, copy a file into it and then delete
+ the file from the directory:
+
+ >>> local.create_dir('/tmp/new_folder')
+ >>> local.copy_file('/tmp/local_fs.dat', '/tmp/new_folder/local_fs.dat')
+ >>> local.delete_file('/tmp/new_folder/local_fs.dat')
+ >>> local.get_file_info('/tmp/new_folder/local_fs.dat')
+ <FileInfo for '/tmp/new_folder/local_fs.dat': type=FileType.NotFound>
+ >>> local.get_file_info('/tmp/new_folder')
+ <FileInfo for '/tmp/new_folder': type=FileType.Directory>
+
+ Move the file:
+
+ >>> local.move('/tmp/local_fs-copy.dat', '/tmp/new_folder/local_fs-copy.dat')
+ >>> local.get_file_info('/tmp/new_folder/local_fs-copy.dat')
+ <FileInfo for '/tmp/new_folder/local_fs-copy.dat': type=FileType.File, size=16>
+ >>> local.get_file_info('/tmp/local_fs-copy.dat')
+ <FileInfo for '/tmp/local_fs-copy.dat': type=FileType.NotFound>
"""
def __init__(self, *, use_mmap=False):
@@ -828,6 +928,45 @@ cdef class SubTreeFileSystem(FileSystem):
The root of the subtree.
base_fs : FileSystem
FileSystem object the operations delegated to.
+
+ Examples
+ --------
+ Create a LocalFileSystem instance:
+
+ >>> from pyarrow import fs
+ >>> local = fs.LocalFileSystem()
+ >>> with local.open_output_stream('/tmp/local_fs.dat') as stream:
+ ... stream.write(b'data')
+ ...
+ 4
+
+ Create a directory and a SubTreeFileSystem instance:
+
+ >>> local.create_dir('/tmp/sub_tree')
+ >>> subtree = fs.SubTreeFileSystem('/tmp/sub_tree', local)
+
+ Write data into the existing file:
+
+ >>> with subtree.open_append_stream('sub_tree_fs.dat') as f:
+ ... f.write(b'+newly added')
+ ...
+ 12
+
+ Print out the attributes:
+
+ >>> subtree.base_fs
+ <pyarrow._fs.LocalFileSystem object at ...>
+ >>> subtree.base_path
+ '/tmp/sub_tree/'
+
+ Get info for the given directory or given file:
+
+ >>> subtree.get_file_info('')
+ <FileInfo for '': type=FileType.Directory>
+ >>> subtree.get_file_info('sub_tree_fs.dat')
+ <FileInfo for 'sub_tree_fs.dat': type=FileType.File, size=12>
+
+ For usage of the methods see examples for :func:`~pyarrow.fs.LocalFileSystem`.
"""
def __init__(self, base_path, FileSystem base_fs):
diff --git a/python/pyarrow/_hdfs.pyx b/python/pyarrow/_hdfs.pyx
index 7a3b974be4..54e0c0eb13 100644
--- a/python/pyarrow/_hdfs.pyx
+++ b/python/pyarrow/_hdfs.pyx
@@ -53,6 +53,13 @@ cdef class HadoopFileSystem(FileSystem):
extra_conf : dict, default None
Extra key/value pairs for configuration; will override any
hdfs-site.xml properties.
+
+ Examples
+ --------
+ >>> from pyarrow import fs
+ >>> hdfs = fs.HadoopFileSystem(host, port, user=user, kerb_ticket=ticket_cache_path) # doctest: +SKIP
+
+ For usage of the methods see examples for :func:`~pyarrow.fs.LocalFileSystem`.
"""
cdef:
diff --git a/python/pyarrow/_s3fs.pyx b/python/pyarrow/_s3fs.pyx
index c71acc4784..f128136de1 100644
--- a/python/pyarrow/_s3fs.pyx
+++ b/python/pyarrow/_s3fs.pyx
@@ -162,6 +162,17 @@ cdef class S3FileSystem(FileSystem):
allow_bucket_deletion : bool, default False
Whether to allow DeleteDir at the bucket-level. This option may also be
passed in a URI query parameter.
+
+ Examples
+ --------
+ >>> from pyarrow import fs
+ >>> s3 = fs.S3FileSystem(region='us-west-2')
+ >>> s3.get_file_info(fs.FileSelector(
+ ... 'power-analysis-ready-datastore/power_901_constants.zarr/FROCEAN', recursive=True
+ ... ))
+ [<FileInfo for 'power-analysis-ready-datastore/power_901_constants.zarr/FROCEAN/.zarray...
+
+ For usage of the methods see examples for :func:`~pyarrow.fs.LocalFileSystem`.
"""
cdef: