You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by ko...@apache.org on 2021/12/18 01:41:21 UTC
[avro] branch master updated: AVRO-3721 Support Python 3.10; Fix Annotations (#1432)
This is an automated email from the ASF dual-hosted git repository.
kojiromike pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/avro.git
The following commit(s) were added to refs/heads/master by this push:
new a946ea7 AVRO-3721 Support Python 3.10; Fix Annotations (#1432)
a946ea7 is described below
commit a946ea706b933606540bc2afae4b81cf14cdbeb0
Author: Michael A. Smith <mi...@smith-li.com>
AuthorDate: Fri Dec 17 20:41:16 2021 -0500
AVRO-3721 Support Python 3.10; Fix Annotations (#1432)
* Add Python 3.10 to Tox to test the latest version of Python.
* Fix Type Checks
* Implement Github Actions for Python 3.10
* Add Python 3.10 in Dockerfile, BUILD.md, and setup.cfg.
---
.github/workflows/test-lang-py.yml | 6 ++++--
BUILD.md | 2 +-
lang/py/avro/__main__.py | 5 ++++-
lang/py/avro/datafile.py | 12 ++++++------
lang/py/avro/io.py | 14 +++++++-------
lang/py/setup.cfg | 1 +
lang/py/tox.ini | 1 +
share/docker/Dockerfile | 9 +++++----
8 files changed, 29 insertions(+), 21 deletions(-)
diff --git a/.github/workflows/test-lang-py.yml b/.github/workflows/test-lang-py.yml
index 19522c0..b8f9800 100644
--- a/.github/workflows/test-lang-py.yml
+++ b/.github/workflows/test-lang-py.yml
@@ -36,6 +36,7 @@ jobs:
fail-fast: false
matrix:
python:
+ - '3.10'
- '3.9'
- '3.8'
- '3.7'
@@ -64,11 +65,11 @@ jobs:
python3 -m pip install --upgrade pip setuptools tox-wheel
- name: Lint
- if: ${{ matrix.python == '3.9' }}
+ if: ${{ matrix.python == '3.10' }}
run: python3 -m tox -e lint
- name: Typechecks
- if: ${{ matrix.python == '3.9' }}
+ if: ${{ matrix.python == '3.10' }}
run: python3 -m tox -e typechecks
- name: Test
@@ -81,6 +82,7 @@ jobs:
fail-fast: false
matrix:
python:
+ - '3.10'
- '3.9'
- '3.8'
- '3.7'
diff --git a/BUILD.md b/BUILD.md
index c09994e..196456f 100644
--- a/BUILD.md
+++ b/BUILD.md
@@ -6,7 +6,7 @@ The following packages must be installed before Avro can be built:
- Java: JDK 1.8, Maven 3 or better, protobuf-compile
- PHP: php7, phpunit, php7-gmp
- - Python 3: 3.5 or greater
+ - Python 3: 3.6 or greater
- C: gcc, cmake, asciidoc, source-highlight, Jansson, pkg-config
- C++: cmake 3.7.2 or greater, g++, flex, bison, libboost-dev
- C#: .NET Core 2.2 SDK
diff --git a/lang/py/avro/__main__.py b/lang/py/avro/__main__.py
index 423de59..44fda88 100755
--- a/lang/py/avro/__main__.py
+++ b/lang/py/avro/__main__.py
@@ -171,7 +171,10 @@ def convert_union(value: str, field: avro.schema.Field) -> Union[int, float, str
def iter_csv(info: IO[AnyStr], schema: avro.schema.RecordSchema) -> Generator[Dict[str, object], None, None]:
header = [field.name for field in schema.fields]
- for row in csv.reader(getattr(i, "decode", lambda: i)() for i in info):
+ # If i is bytes, decode into a string.
+ # If i is a string, no need to decode.
+ csv_data = (cast(str, getattr(i, "decode", lambda: i)()) for i in info)
+ for row in csv.reader(csv_data):
values = [convert(v, f) for v, f in zip(row, schema.fields)]
yield dict(zip(header, values))
diff --git a/lang/py/avro/datafile.py b/lang/py/avro/datafile.py
index 11d9a3c..d39a911 100644
--- a/lang/py/avro/datafile.py
+++ b/lang/py/avro/datafile.py
@@ -160,7 +160,7 @@ class DataFileWriter(_DataFileMetadata):
_datum_writer: avro.io.DatumWriter
_encoder: avro.io.BinaryEncoder
_header_written: bool
- _writer: BinaryIO
+ _writer: IO[bytes]
block_count: int
sync_marker: bytes
@@ -170,7 +170,7 @@ class DataFileWriter(_DataFileMetadata):
"""If the schema is not present, presume we're appending."""
if hasattr(writer, "mode") and "b" not in writer.mode:
warnings.warn(avro.errors.AvroWarning(f"Writing binary data to a writer {writer!r} that's opened for text"))
- bytes_writer = getattr(writer, "buffer", writer)
+ bytes_writer = cast(IO[bytes], getattr(writer, "buffer", writer))
self._writer = bytes_writer
self._encoder = avro.io.BinaryEncoder(bytes_writer)
self._datum_writer = datum_writer
@@ -202,7 +202,7 @@ class DataFileWriter(_DataFileMetadata):
self.datum_writer.writers_schema = writers_schema
@property
- def writer(self) -> BinaryIO:
+ def writer(self) -> IO[bytes]:
return self._writer
@property
@@ -307,7 +307,7 @@ class DataFileReader(_DataFileMetadata):
_datum_reader: avro.io.DatumReader
_file_length: int
_raw_decoder: avro.io.BinaryDecoder
- _reader: BinaryIO
+ _reader: IO[bytes]
block_count: int
sync_marker: bytes
@@ -317,7 +317,7 @@ class DataFileReader(_DataFileMetadata):
def __init__(self, reader: IO[AnyStr], datum_reader: avro.io.DatumReader) -> None:
if hasattr(reader, "mode") and "b" not in reader.mode:
warnings.warn(avro.errors.AvroWarning(f"Reader binary data from a reader {reader!r} that's opened for text"))
- bytes_reader = getattr(reader, "buffer", reader)
+ bytes_reader = cast(IO[bytes], getattr(reader, "buffer", reader))
self._reader = bytes_reader
self._raw_decoder = avro.io.BinaryDecoder(bytes_reader)
self._datum_decoder = None # Maybe reset at every block.
@@ -337,7 +337,7 @@ class DataFileReader(_DataFileMetadata):
return self
@property
- def reader(self) -> BinaryIO:
+ def reader(self) -> IO[bytes]:
return self._reader
@property
diff --git a/lang/py/avro/io.py b/lang/py/avro/io.py
index d8b0f94..59628b0 100644
--- a/lang/py/avro/io.py
+++ b/lang/py/avro/io.py
@@ -90,7 +90,7 @@ import decimal
import struct
import warnings
from typing import (
- BinaryIO,
+ IO,
Deque,
Generator,
Iterable,
@@ -206,16 +206,16 @@ _ITERATORS["error"] = _ITERATORS["request"] = _ITERATORS["record"]
class BinaryDecoder:
"""Read leaf values."""
- _reader: BinaryIO
+ _reader: IO[bytes]
- def __init__(self, reader: BinaryIO) -> None:
+ def __init__(self, reader: IO[bytes]) -> None:
"""
reader is a Python object on which we can call read, seek, and tell.
"""
self._reader = reader
@property
- def reader(self) -> BinaryIO:
+ def reader(self) -> IO[bytes]:
return self._reader
def read(self, n: int) -> bytes:
@@ -410,16 +410,16 @@ class BinaryDecoder:
class BinaryEncoder:
"""Write leaf values."""
- _writer: BinaryIO
+ _writer: IO[bytes]
- def __init__(self, writer: BinaryIO) -> None:
+ def __init__(self, writer: IO[bytes]) -> None:
"""
writer is a Python object on which we can call write.
"""
self._writer = writer
@property
- def writer(self) -> BinaryIO:
+ def writer(self) -> IO[bytes]:
return self._writer
def write(self, datum: bytes) -> None:
diff --git a/lang/py/setup.cfg b/lang/py/setup.cfg
index 487b322..adbeb0f 100644
--- a/lang/py/setup.cfg
+++ b/lang/py/setup.cfg
@@ -37,6 +37,7 @@ classifiers =
Programming Language :: Python :: 3.7
Programming Language :: Python :: 3.8
Programming Language :: Python :: 3.9
+ Programming Language :: Python :: 3.10
Development Status :: 5 - Production/Stable
[bdist_wheel]
diff --git a/lang/py/tox.ini b/lang/py/tox.ini
index bbd42d6..ab2ceaa 100644
--- a/lang/py/tox.ini
+++ b/lang/py/tox.ini
@@ -24,6 +24,7 @@ envlist =
py37
py38
py39
+ py310
pypy3.6
pypy3.7
diff --git a/share/docker/Dockerfile b/share/docker/Dockerfile
index 619861e..7b8ce55 100644
--- a/share/docker/Dockerfile
+++ b/share/docker/Dockerfile
@@ -66,7 +66,11 @@ RUN apt-get -qqy update \
valgrind \
vim \
wget \
- python3.6 python3.7 python3.8 python3.9 \
+ python3.6 \
+ python3.7 \
+ python3.8 \
+ python3.9 \
+ python3.10 \
&& apt-get -qqy clean
# Install PHP
@@ -194,7 +198,4 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --de
ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
ENV PATH $JAVA_HOME/bin:$PATH
-
-RUN apt-get -qqy install python3.6 python3.7 python3.9
-
CMD ["/bin/bash", "-i"]