You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by bl...@apache.org on 2022/07/10 22:42:38 UTC
[iceberg] branch master updated: Python: Reenable mypy (#5171)
This is an automated email from the ASF dual-hosted git repository.
blue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new d1d708757 Python: Reenable mypy (#5171)
d1d708757 is described below
commit d1d70875714ada876cc651be6d1b3a1d7483a0da
Author: Fokko Driesprong <fo...@apache.org>
AuthorDate: Mon Jul 11 00:42:32 2022 +0200
Python: Reenable mypy (#5171)
---
python/.pre-commit-config.yaml | 1 +
python/pyiceberg/io/base.py | 13 ++-----------
python/pyiceberg/io/memory.py | 9 +++------
python/pyproject.toml | 22 +++++++++++++++++++---
python/tests/avro/test_decoder.py | 5 +++--
python/tests/conftest.py | 2 +-
python/tests/io/test_io_base.py | 2 +-
python/tests/test_schema.py | 4 ++--
8 files changed, 32 insertions(+), 26 deletions(-)
diff --git a/python/.pre-commit-config.yaml b/python/.pre-commit-config.yaml
index 235e44f2d..0b01ae1af 100644
--- a/python/.pre-commit-config.yaml
+++ b/python/.pre-commit-config.yaml
@@ -40,6 +40,7 @@ repos:
rev: v0.961
hooks:
- id: mypy
+ args: [--config=python/pyproject.toml]
- repo: https://github.com/hadialqattan/pycln
rev: v1.3.4
hooks:
diff --git a/python/pyiceberg/io/base.py b/python/pyiceberg/io/base.py
index 458a3d591..3098102b9 100644
--- a/python/pyiceberg/io/base.py
+++ b/python/pyiceberg/io/base.py
@@ -22,7 +22,6 @@ as check if a file exists. An implementation of the FileIO abstract base class i
for returning an InputFile instance, an OutputFile instance, and deleting a file given
its location.
"""
-
from abc import ABC, abstractmethod
from io import SEEK_SET
from typing import Protocol, Union, runtime_checkable
@@ -41,17 +40,13 @@ class InputStream(Protocol):
...
@abstractmethod
- def seek(self, offset: int, whence: int = SEEK_SET) -> None:
+ def seek(self, offset: int, whence: int = SEEK_SET) -> int:
...
@abstractmethod
def tell(self) -> int:
...
- @abstractmethod
- def closed(self) -> bool:
- ...
-
@abstractmethod
def close(self) -> None:
...
@@ -66,11 +61,7 @@ class OutputStream(Protocol): # pragma: no cover
"""
@abstractmethod
- def write(self, b: bytes) -> None:
- ...
-
- @abstractmethod
- def closed(self) -> bool:
+ def write(self, b: bytes) -> int:
...
@abstractmethod
diff --git a/python/pyiceberg/io/memory.py b/python/pyiceberg/io/memory.py
index 082a589f1..73f0d5125 100644
--- a/python/pyiceberg/io/memory.py
+++ b/python/pyiceberg/io/memory.py
@@ -36,8 +36,6 @@ class MemoryInputStream(InputStream):
>>> stream.read(4)
b'1925'
>>> stream.close()
- >>> stream.closed()
- True
"""
buffer: bytes
@@ -54,7 +52,7 @@ class MemoryInputStream(InputStream):
self.pos += size
return b
- def seek(self, offset: int, whence: int = SEEK_SET) -> None:
+ def seek(self, offset: int, whence: int = SEEK_SET) -> int:
if whence == SEEK_SET:
self.pos = offset
elif whence == SEEK_CUR:
@@ -64,11 +62,10 @@ class MemoryInputStream(InputStream):
else:
raise ValueError(f"Unknown whence {offset}")
- def tell(self) -> int:
return self.pos
- def closed(self) -> bool:
- return not hasattr(self, "buffer")
+ def tell(self) -> int:
+ return self.pos
def close(self) -> None:
del self.buffer
diff --git a/python/pyproject.toml b/python/pyproject.toml
index 6500a5dbf..a3e35bfc9 100644
--- a/python/pyproject.toml
+++ b/python/pyproject.toml
@@ -84,15 +84,31 @@ warn_redundant_casts = true
warn_unreachable = true
[[tool.mypy.overrides]]
-module = "mypy-pyarrow.*"
+module = "pyarrow.*"
ignore_missing_imports = true
[[tool.mypy.overrides]]
-module = "mypy-snappy.*"
+module = "snappy.*"
ignore_missing_imports = true
[[tool.mypy.overrides]]
-module = "mypy-zstandard.*"
+module = "zstandard.*"
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "pydantic.*"
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "pytest.*"
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "fastavro.*"
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "mmh3.*"
ignore_missing_imports = true
[tool.coverage.run]
diff --git a/python/tests/avro/test_decoder.py b/python/tests/avro/test_decoder.py
index dbf940eb5..0616d4931 100644
--- a/python/tests/avro/test_decoder.py
+++ b/python/tests/avro/test_decoder.py
@@ -106,14 +106,15 @@ class OneByteAtATimeInputStream(InputStream):
self.pos += 1
return int.to_bytes(1, self.pos, byteorder="little")
- def seek(self, offset: int, whence: int = SEEK_SET) -> None:
+ def seek(self, offset: int, whence: int = SEEK_SET) -> int:
pass
def tell(self) -> int:
pass
+ @property
def closed(self) -> bool:
- pass
+ return False
def close(self) -> None:
pass
diff --git a/python/tests/conftest.py b/python/tests/conftest.py
index 73ae489a3..1b004ec8f 100644
--- a/python/tests/conftest.py
+++ b/python/tests/conftest.py
@@ -828,7 +828,7 @@ class LocalOutputFile(OutputFile):
def create(self, overwrite: bool = False) -> OutputStream:
output_file = open(self._path, "wb" if overwrite else "xb")
- if not isinstance(output_file, OutputStream):
+ if not issubclass(type(output_file), OutputStream):
raise TypeError("Object returned from LocalOutputFile.create(...) does not match the OutputStream protocol.")
return output_file
diff --git a/python/tests/io/test_io_base.py b/python/tests/io/test_io_base.py
index a7a049ecd..3165c4099 100644
--- a/python/tests/io/test_io_base.py
+++ b/python/tests/io/test_io_base.py
@@ -104,7 +104,7 @@ class LocalOutputFile(OutputFile):
def create(self, overwrite: bool = False) -> OutputStream:
output_file = open(self.parsed_location.path, "wb" if overwrite else "xb")
- if not isinstance(output_file, OutputStream):
+ if not issubclass(type(output_file), OutputStream):
raise TypeError("Object returned from LocalOutputFile.create(...) does not match the OutputStream protocol.")
return output_file
diff --git a/python/tests/test_schema.py b/python/tests/test_schema.py
index 639fa0076..3d61323f0 100644
--- a/python/tests/test_schema.py
+++ b/python/tests/test_schema.py
@@ -16,7 +16,7 @@
# under the License.
from textwrap import dedent
-from typing import Any, Dict
+from typing import Any, Dict, Optional
import pytest
@@ -388,7 +388,7 @@ def test_build_position_accessors(table_schema_nested):
def test_build_position_accessors_with_struct(table_schema_nested: Schema):
class TestStruct(StructProtocol):
- def __init__(self, pos: Dict[int, Any] = None):
+ def __init__(self, pos: Optional[Dict[int, Any]] = None):
self._pos: Dict[int, Any] = pos or {}
def set(self, pos: int, value) -> None: