You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by rs...@apache.org on 2020/01/29 08:54:19 UTC
[avro] branch branch-1.9 updated (248afff -> 4006a76)
This is an automated email from the ASF dual-hosted git repository.
rskraba pushed a change to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/avro.git.
from 248afff [AVRO-2663] Bug with nested record
new 6216bba AVRO-2391 Refactor Py3 Setup (#514)
new a494515 AVRO-2388: Drop Python 3.4 Support (#513)
new 3fc05a3 AVRO-2413: Refactor io.Validate to use a mapping (#516)
new 8302424 AVRO-1928: Simplify Python float/double encoding (#528)
new a627959 AVRO-2104: Schema normalization and fingerprinting for Python 3 (#536)
new 702770f AVRO-2445: Remove SimpleJSON (#566)
new 62e2db9 AVRO-2445: Remove StoppableHTTPServer Polyfill (#567)
new 138f55f AVRO-2445: Remove context manager avoidance (#568)
new 89ed672 AVRO-2410: Framework for Linting (#569)
new a9f05b4 AVRO-2576: Fix Premature Import pycodestyle (#664)
new b824cf3 AVRO-2577: Fix Bare Excepts (#665)
new 78f26cc AVRO-2578: Support Both Capitalizations of Parse (#666)
new 1c190fc AVRO-2578: Deprecate Parse in 1.9.2.
new e772419 AVRO-2580: Enforce Logical Type and Literal Type Match (#668)
new 083801f AVRO-2590: Dockerfile Fails to Fetch Microsoft Dist (#677)
new d9ef637 Undo some language tool bumps.
new 8de8fb3 AVRO-2593: Ensure exfile is Always Defined
new 075a12a AVRO-2601: zstandard in extras_require
new e9eee38 AVRO-1788: Remove Obsolete Python < 2.7 Syntax (#683)
new 76611fc AVRO-2603: Clean Up Imports (#684)
new 2cfe522 AVRO-2605: Remove Use of Filter Function (#688)
new f6aee38 AVRO-2456: Add interop test for the snappy and zstd codec (#573)
new 4006a76 AVRO-2595: Remove Unusable Py3 Txipc (#716)
The 23 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.editorconfig | 6 +-
LICENSE.txt | 25 -
build.sh | 12 +-
lang/c++/build.sh | 6 +-
lang/c/build.sh | 7 +-
lang/csharp/build.sh | 8 +-
lang/java/avro/pom.xml | 32 +-
lang/java/build.sh | 70 +-
lang/java/checkstyle.xml | 2 -
lang/java/ipc/pom.xml | 8 +
lang/java/pom.xml | 1 -
lang/js/build.sh | 7 +-
lang/perl/build.sh | 5 +-
lang/php/build.sh | 6 +-
lang/py/.gitignore | 2 +
lang/py/build.sh | 66 +-
lang/py/lib/simplejson/LICENSE.txt | 19 -
lang/py/lib/simplejson/__init__.py | 318 ----
lang/py/lib/simplejson/_speedups.c | 2329 -------------------------
lang/py/lib/simplejson/decoder.py | 354 ----
lang/py/lib/simplejson/encoder.py | 440 -----
lang/py/lib/simplejson/scanner.py | 65 -
lang/py/lib/simplejson/tool.py | 37 -
lang/py/scripts/avro | 30 +-
.editorconfig => lang/py/setup.cfg | 34 +-
lang/py/setup.py | 78 +-
lang/py/src/avro/__init__.py | 10 +-
lang/py/src/avro/constants.py | 13 +-
lang/py/src/avro/datafile.py | 58 +-
lang/py/src/avro/io.py | 231 ++-
lang/py/src/avro/ipc.py | 34 +-
lang/py/src/avro/protocol.py | 22 +-
lang/py/src/avro/schema.py | 108 +-
lang/py/src/avro/tether/__init__.py | 15 +-
lang/py/src/avro/tether/tether_task.py | 55 +-
lang/py/src/avro/tether/tether_task_runner.py | 73 +-
lang/py/src/avro/tether/util.py | 39 +-
lang/py/src/avro/timezones.py | 13 +-
lang/py/src/avro/tool.py | 79 +-
lang/py/src/avro/txipc.py | 22 +-
lang/py/{src/avro => test}/__init__.py | 10 +-
lang/py/test/av_bench.py | 12 +-
lang/py/test/gen_interop_data.py | 45 +-
lang/py/test/mock_tether_parent.py | 41 +-
lang/py/test/sample_http_client.py | 12 +-
lang/py/test/sample_http_server.py | 8 +-
lang/py/test/set_avro_test_path.py | 18 +-
lang/py/test/test_datafile.py | 90 +-
lang/py/test/test_datafile_interop.py | 46 +-
lang/py/test/test_io.py | 71 +-
lang/py/test/test_ipc.py | 9 +-
lang/py/test/test_protocol.py | 92 +-
lang/py/test/test_schema.py | 782 ++++-----
lang/py/test/test_script.py | 46 +-
lang/py/test/test_tether_task.py | 42 +-
lang/py/test/test_tether_task_runner.py | 67 +-
lang/py/test/test_tether_word_count.py | 56 +-
lang/py/test/txsample_http_client.py | 13 +-
lang/py/test/txsample_http_server.py | 11 +-
lang/py/test/word_count_task.py | 52 +-
lang/py3/.gitignore | 1 +
lang/{ruby/Gemfile => py3/MANIFEST.in} | 13 +-
lang/py3/avro/datafile.py | 8 +-
lang/py3/avro/io.py | 108 +-
lang/py3/avro/ipc.py | 17 +-
lang/py3/avro/protocol.py | 13 +-
lang/py3/avro/schema.py | 25 +-
lang/py3/avro/schemanormalization.py | 224 +++
lang/py3/avro/tests/av_bench.py | 3 +-
lang/py3/avro/tests/gen_interop_data.py | 29 +-
lang/py3/avro/tests/run_tests.py | 2 +-
lang/py3/avro/tests/sample_http_client.py | 7 +-
lang/py3/avro/tests/sample_http_server.py | 7 +-
lang/py3/avro/tests/test_datafile.py | 22 +-
lang/py3/avro/tests/test_enum.py | 1 +
lang/py3/avro/tests/test_io.py | 31 +-
lang/py3/avro/tests/test_ipc.py | 6 +-
lang/py3/avro/tests/test_normalization.py | 2029 +++++++++++++++++++++
lang/py3/avro/tests/test_protocol.py | 38 +-
lang/py3/avro/tests/test_schema.py | 36 +-
lang/py3/avro/tests/test_script.py | 3 +-
lang/py3/avro/tests/txsample_http_client.py | 108 --
lang/py3/avro/tests/txsample_http_server.py | 72 -
lang/py3/avro/tool.py | 45 +-
lang/py3/avro/txipc.py | 224 ---
lang/py3/build.sh | 59 +-
lang/py3/scripts/avro | 2 +-
lang/py3/setup.cfg | 78 +
lang/py3/setup.py | 268 +--
lang/ruby/build.sh | 39 +-
lang/ruby/interop/test_interop.rb | 10 +-
pom.xml | 44 +-
share/docker/Dockerfile | 177 +-
share/docker/run-tests.sh | 2 +-
94 files changed, 4174 insertions(+), 5839 deletions(-)
delete mode 100644 lang/py/lib/simplejson/LICENSE.txt
delete mode 100644 lang/py/lib/simplejson/__init__.py
delete mode 100644 lang/py/lib/simplejson/_speedups.c
delete mode 100644 lang/py/lib/simplejson/decoder.py
delete mode 100644 lang/py/lib/simplejson/encoder.py
delete mode 100644 lang/py/lib/simplejson/scanner.py
delete mode 100644 lang/py/lib/simplejson/tool.py
copy .editorconfig => lang/py/setup.cfg (64%)
copy lang/py/{src/avro => test}/__init__.py (89%)
copy lang/{ruby/Gemfile => py3/MANIFEST.in} (79%)
create mode 100644 lang/py3/avro/schemanormalization.py
create mode 100644 lang/py3/avro/tests/test_normalization.py
delete mode 100644 lang/py3/avro/tests/txsample_http_client.py
delete mode 100644 lang/py3/avro/tests/txsample_http_server.py
delete mode 100644 lang/py3/avro/txipc.py
create mode 100644 lang/py3/setup.cfg
[avro] 21/23: AVRO-2605: Remove Use of Filter Function (#688)
Posted by rs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rskraba pushed a commit to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/avro.git
commit 2cfe522b08d07cbdfc037d6dbee3936601483616
Author: Michael A. Smith <mi...@smith-li.com>
AuthorDate: Sat Oct 26 07:50:14 2019 -0400
AVRO-2605: Remove Use of Filter Function (#688)
---
lang/py/test/set_avro_test_path.py | 11 ++++-------
1 file changed, 4 insertions(+), 7 deletions(-)
diff --git a/lang/py/test/set_avro_test_path.py b/lang/py/test/set_avro_test_path.py
index fd395da..29b666c 100644
--- a/lang/py/test/set_avro_test_path.py
+++ b/lang/py/test/set_avro_test_path.py
@@ -35,13 +35,10 @@ build of AVRO is higher on the path then any installed eggs.
from __future__ import absolute_import, division, print_function
-import os
+import os.path
import sys
-# determine the build directory and then make sure all paths that start with the
+# Make sure all paths that start with the
# build directory are at the top of the path
-builddir=os.path.split(os.path.split(__file__)[0])[0]
-bpaths=filter(lambda s:s.startswith(builddir), sys.path)
-
-for p in bpaths:
- sys.path.insert(0,p)
+builddir = os.path.dirname(os.path.dirname(__file__))
+sys.path[:0] = [p for p in sys.path if p.startswith(builddir)]
[avro] 17/23: AVRO-2593: Ensure exfile is Always Defined
Posted by rs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rskraba pushed a commit to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/avro.git
commit 8de8fb33d7a14ba276d360db22144178aeaefa34
Author: Michael A. Smith <mi...@smith-li.com>
AuthorDate: Sat Oct 19 20:34:45 2019 -0400
AVRO-2593: Ensure exfile is Always Defined
Fixes a bug where when a test fails the cleanup code tries to access a variable that might not have been defined when the error occurred.
---
lang/py/test/test_tether_word_count.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/lang/py/test/test_tether_word_count.py b/lang/py/test/test_tether_word_count.py
index cc34f2b..d2f1858 100644
--- a/lang/py/test/test_tether_word_count.py
+++ b/lang/py/test/test_tether_word_count.py
@@ -98,6 +98,7 @@ class TestTetherWordCount(unittest.TestCase):
import inspect
proc=None
+ exfile = None
try:
@@ -207,7 +208,7 @@ python -m avro.tether.tether_task_runner word_count_task.WordCountTask
proc.kill()
if os.path.exists(base_dir):
shutil.rmtree(base_dir)
- if os.path.exists(exfile):
+ if exfile is not None and os.path.exists(exfile):
os.remove(exfile)
if __name__== "__main__":
[avro] 15/23: AVRO-2590: Dockerfile Fails to Fetch Microsoft Dist
(#677)
Posted by rs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rskraba pushed a commit to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/avro.git
commit 083801f93998508bc28c0a6e0b18bcf268909361
Author: Michael A. Smith <mi...@smith-li.com>
AuthorDate: Fri Oct 18 09:59:39 2019 -0400
AVRO-2590: Dockerfile Fails to Fetch Microsoft Dist (#677)
* AVRO-2590: Install .NET with Ubuntu 16 Deb
* AVRO-2590: Remove Build Caches
---
share/docker/Dockerfile | 183 +++++++++++++++++++++++++++++-------------------
1 file changed, 110 insertions(+), 73 deletions(-)
diff --git a/share/docker/Dockerfile b/share/docker/Dockerfile
index 726d321..926b270 100644
--- a/share/docker/Dockerfile
+++ b/share/docker/Dockerfile
@@ -18,84 +18,116 @@
# See BUILD.txt.
FROM openjdk:8
-
WORKDIR /root
+CMD ["/avro/share/docker/run-tests.sh"]
+ENV APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=isolemnlysweariamuptonogood \
+ DEBIAN_FRONTEND=noninteractive
-SHELL ["/bin/bash", "-o", "pipefail", "-c"]
-
-# Add the repository for node.js 6.x
-RUN curl -sL https://deb.nodesource.com/setup_6.x | bash -
-
-# Register Microsoft key and feed for .NET SDK
-# https://dotnet.microsoft.com/download/linux-package-manager/debian8/sdk-current
-RUN wget -qO- https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.asc.gpg && \
- mv microsoft.asc.gpg /etc/apt/trusted.gpg.d/ && \
- wget -q https://packages.microsoft.com/config/debian/9/prod.list && \
- mv prod.list /etc/apt/sources.list.d/microsoft-prod.list && \
- chown root:root /etc/apt/trusted.gpg.d/microsoft.asc.gpg && \
- chown root:root /etc/apt/sources.list.d/microsoft-prod.list
-
-RUN curl https://packages.sury.org/php/apt.gpg | apt-key add --no-tty - && \
- echo "deb https://packages.sury.org/php/ $(lsb_release -sc) main" > /etc/apt/sources.list.d/php.list
+# Install dependencies from vanilla system packages
+RUN apt-get -qqy update \
+ && apt-get -qqy install --no-install-recommends ant \
+ apt-transport-https \
+ asciidoc \
+ bison \
+ bzip2 \
+ bundler \
+ cmake \
+ curl \
+ doxygen \
+ flex \
+ g++ \
+ gcc \
+ git \
+ libboost-all-dev \
+ libfontconfig1-dev \
+ libfreetype6-dev \
+ libglib2.0-dev \
+ libjansson-dev \
+ libsnappy-dev \
+ libsnappy1v5 \
+ make \
+ maven \
+ perl \
+ python \
+ python-pip \
+ python-setuptools \
+ python-snappy \
+ python-wheel \
+ python3-pip \
+ python3-setuptools \
+ python3-snappy \
+ python3-wheel \
+ rake \
+ ruby \
+ ruby-dev \
+ source-highlight \
+ subversion \
+ valgrind \
+ && apt-get -qqy clean \
+ && rm -rf /var/lib/apt/lists
-# Install dependencies from packages
-RUN apt-get -qq update && \
- apt-get -qq install --no-install-recommends -y \
- ant \
- asciidoc \
- bison \
- bzip2 \
- cmake \
- curl \
- dotnet-sdk-2.2 \
- doxygen \
- flex \
- g++ \
- gcc \
- git \
- libboost-all-dev \
- libfontconfig1-dev \
- libfreetype6-dev \
- libglib2.0-dev \
- libjansson-dev \
- libsnappy-dev \
- libsnappy1v5 \
- make \
- maven \
- nodejs \
- perl \
- php5.6 \
- php5.6-gmp \
- pycodestyle \
- python \
- python-pip \
- python-setuptools \
- python-snappy \
- python-wheel \
- python3-pip \
- python3-setuptools \
- python3-snappy \
- python3-wheel \
- rake \
- ruby \
- ruby-dev \
- source-highlight \
- subversion \
- valgrind && \
- apt-get -qq clean && \
- rm -rf /var/lib/apt/lists/*
+# Install nodejs 6
+RUN curl -sSL https://deb.nodesource.com/setup_6.x \
+ | bash - \
+ && apt-get -qqy install nodejs \
+ && apt-get -qqy clean \
+ && rm -rf /var/lib/apt/lists \
+ && npm install -g grunt-cli \
+ && npm cache clean
# Install Perl modules
RUN curl -L https://cpanmin.us | perl - --self-upgrade && \
- cpanm --mirror https://www.cpan.org/ install \
- Module::Install Module::Install::ReadmeFromPod \
+ cpanm --mirror https://www.cpan.org/ install Module::Install Module::Install::ReadmeFromPod \
Module::Install::Repository \
Math::BigInt JSON::XS Try::Tiny Regexp::Common Encode \
IO::String Object::Tiny Compress::Zlib Test::More \
- Test::Exception Test::Pod
+ Test::Exception Test::Pod Perl::Critic Compress::Zstd
+
+# Install PHP
+# TODO: Remove php5-related packages when we drop PHP 5.6 support in the future
+# TODO: Consolidate the php versions into a single one (7.x) when we drop PHP 5.6 support in the future
+ENV PHP5_VERSION=5.6 \
+ PHP7_VERSION=7.1
+RUN curl -sSL https://packages.sury.org/php/apt.gpg \
+ | apt-key add --no-tty - \
+ && echo "deb https://packages.sury.org/php/ $(lsb_release -sc) main" > /etc/apt/sources.list.d/php.list \
+ && apt-get -qqy update \
+ && apt-get -qqy install --no-install-recommends php${PHP5_VERSION} \
+ php${PHP5_VERSION}-gmp \
+ php${PHP7_VERSION} \
+ php${PHP7_VERSION}-gmp \
+ && apt-get -qqy clean \
+ && rm -rf /var/lib/apt/lists
+# Install PHPUnit 5.7, which is the only version that supports both PHP 5.6 and 7.x
+# TODO: use PHPUnit 7.x instead of 5.7 when we drop PHP 5.6 support in the future
+RUN wget -O /usr/local/bin/phpunit https://phar.phpunit.de/phpunit-5.7.phar && chmod +x /usr/local/bin/phpunit
-# Install PHPUnit
-RUN wget -O /usr/local/bin/phpunit https://phar.phpunit.de/phpunit-5.6.phar && chmod +x /usr/local/bin/phpunit
+RUN curl -sSL https://phar.phpunit.de/phpunit-5.7.phar > /usr/local/bin/phpunit \
+ && chmod +x /usr/local/bin/phpunit
+
+# Install Perl modules
+RUN curl -sSL https://cpanmin.us \
+ | perl - --self-upgrade \
+ && cpanm --mirror https://www.cpan.org/ install Compress::Zlib \
+ Compress::Zstd \
+ Encode \
+ IO::String \
+ JSON::XS \
+ Math::BigInt \
+ Module::Install \
+ Module::Install::ReadmeFromPod \
+ Module::Install::Repository \
+ Object::Tiny \
+ Perl::Critic \
+ Regexp::Common \
+ Test::Exception \
+ Test::More \
+ Test::Pod \
+ Try::Tiny \
+ && rm -rf .cpanm
+
+# Install Python packages
+ENV PIP_NO_CACHE_DIR=off
# Install Python2 packages
RUN python2 -m pip install --upgrade pip setuptools \
@@ -106,9 +138,14 @@ RUN python3 -m pip install --upgrade pip setuptools \
&& python3 -m pip install zstandard
# Install Ruby modules
-RUN gem install echoe yajl-ruby multi_json snappy
+COPY lang/ruby/Gemfile /tmp
+RUN bundle install --gemfile=/tmp/Gemfile
-# Install global Node modules
-RUN npm install -g grunt-cli
-
-CMD ["/avro/share/docker/run-tests.sh"]
+# Install .NET SDK
+RUN curl -sSLO https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb \
+ && dpkg -i packages-microsoft-prod.deb \
+ && rm packages-microsoft-prod.deb \
+ && apt-get -qqy update \
+ && apt-get -qqy install --no-install-recommends dotnet-sdk-2.2 \
+ && apt-get -qqy clean \
+ && rm -rf /var/lib/apt/lists
[avro] 11/23: AVRO-2577: Fix Bare Excepts (#665)
Posted by rs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rskraba pushed a commit to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/avro.git
commit b824cf3ec444a2439f0e3ff0eba5871de7342fa4
Author: Michael A. Smith <mi...@smith-li.com>
AuthorDate: Fri Oct 25 17:58:36 2019 -0400
AVRO-2577: Fix Bare Excepts (#665)
* AVRO-2577: Fix Bare Excepts
* AVRO-2577: Don't Count Failure as Success
---
lang/py/setup.cfg | 2 +-
lang/py/src/avro/datafile.py | 53 ++++++++++++-----
lang/py/src/avro/protocol.py | 39 ++++++-------
lang/py/src/avro/schema.py | 7 ++-
lang/py/test/sample_http_client.py | 2 +-
lang/py/test/test_protocol.py | 90 +++++++++++++++--------------
lang/py/test/txsample_http_client.py | 2 +-
lang/py3/avro/tests/sample_http_client.py | 2 +-
lang/py3/avro/tests/test_protocol.py | 15 +++--
lang/py3/avro/tests/txsample_http_client.py | 2 +-
lang/py3/setup.cfg | 2 +-
11 files changed, 123 insertions(+), 93 deletions(-)
diff --git a/lang/py/setup.cfg b/lang/py/setup.cfg
index 1f6fbfa..aaa8a29 100644
--- a/lang/py/setup.cfg
+++ b/lang/py/setup.cfg
@@ -20,6 +20,6 @@ known_third_party=zope
[pycodestyle]
exclude = .eggs,build
-ignore = E101,E111,E114,E121,E122,E124,E125,E126,E127,E128,E129,E201,E202,E203,E222,E226,E225,E231,E241,E251,E261,E262,E265,E266,E301,E302,E303,E305,E306,E402,E501,E701,E703,E704,E711,E722,W191,W291,W292,W293,W391,W503,W504,W601
+ignore = E101,E111,E114,E121,E122,E124,E125,E126,E127,E128,E129,E201,E202,E203,E222,E226,E225,E231,E241,E251,E261,E262,E265,E266,E301,E302,E303,E305,E306,E402,E501,E701,E703,E704,E711,W191,W291,W292,W293,W391,W503,W504,W601
max-line-length = 150
statistics = True
diff --git a/lang/py/src/avro/datafile.py b/lang/py/src/avro/datafile.py
index 772d949..75a8e4a 100644
--- a/lang/py/src/avro/datafile.py
+++ b/lang/py/src/avro/datafile.py
@@ -1,3 +1,6 @@
+#!/usr/bin/env python
+
+##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -13,9 +16,13 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-"""
-Read/Write Avro File Object Containers.
-"""
+
+"""Read/Write Avro File Object Containers."""
+
+from __future__ import absolute_import, division, print_function
+
+import os
+import random
import zlib
from avro import io, schema
@@ -29,6 +36,11 @@ try:
has_snappy = True
except ImportError:
has_snappy = False
+try:
+ import zstandard as zstd
+ has_zstandard = True
+except ImportError:
+ has_zstandard = False
#
# Constants
#
@@ -48,6 +60,8 @@ META_SCHEMA = schema.parse("""\
VALID_CODECS = ['null', 'deflate']
if has_snappy:
VALID_CODECS.append('snappy')
+if has_zstandard:
+ VALID_CODECS.append('zstandard')
VALID_ENCODINGS = ['binary'] # not used yet
CODEC_KEY = "avro.codec"
@@ -99,7 +113,7 @@ class DataFileWriter(object):
else:
# open writer for reading to collect metadata
dfr = DataFileReader(writer, io.DatumReader())
-
+
# TODO(hammer): collect arbitrary metadata
# collect metadata
self._sync_marker = dfr.sync_marker
@@ -171,6 +185,9 @@ class DataFileWriter(object):
elif self.get_meta(CODEC_KEY) == 'snappy':
compressed_data = snappy.compress(uncompressed_data)
compressed_data_length = len(compressed_data) + 4 # crc32
+ elif self.get_meta(CODEC_KEY) == 'zstandard':
+ compressed_data = zstd.ZstdCompressor().compress(uncompressed_data)
+ compressed_data_length = len(compressed_data)
else:
fail_msg = '"%s" codec is not supported.' % self.get_meta(CODEC_KEY)
raise DataFileException(fail_msg)
@@ -180,7 +197,7 @@ class DataFileWriter(object):
# Write block
self.writer.write(compressed_data)
-
+
# Write CRC32 checksum for Snappy
if self.get_meta(CODEC_KEY) == 'snappy':
self.encoder.write_crc32(uncompressed_data)
@@ -189,7 +206,7 @@ class DataFileWriter(object):
self.writer.write(self.sync_marker)
# reset buffer
- self.buffer_writer.truncate(0)
+ self.buffer_writer.truncate(0)
self.block_count = 0
def append(self, datum):
@@ -229,7 +246,7 @@ class DataFileReader(object):
self._raw_decoder = io.BinaryDecoder(reader)
self._datum_decoder = None # Maybe reset at every block.
self._datum_reader = datum_reader
-
+
# read the header: magic, meta, sync
self._read_header()
@@ -293,7 +310,7 @@ class DataFileReader(object):
def _read_header(self):
# seek to the beginning of the file to get magic block
- self.reader.seek(0, 0)
+ self.reader.seek(0, 0)
# read header into a dict
header = self.datum_reader.read_data(
@@ -332,6 +349,18 @@ class DataFileReader(object):
uncompressed = snappy.decompress(data)
self._datum_decoder = io.BinaryDecoder(StringIO(uncompressed))
self.raw_decoder.check_crc32(uncompressed);
+ elif self.codec == 'zstandard':
+ length = self.raw_decoder.read_long()
+ data = self.raw_decoder.read(length)
+ uncompressed = bytearray()
+ dctx = zstd.ZstdDecompressor()
+ with dctx.stream_reader(StringIO(data)) as reader:
+ while True:
+ chunk = reader.read(16384)
+ if not chunk:
+ break
+ uncompressed.extend(chunk)
+ self._datum_decoder = io.BinaryDecoder(StringIO(uncompressed))
else:
raise DataFileException("Unknown codec: %r" % self.codec)
@@ -360,7 +389,7 @@ class DataFileReader(object):
else:
self._read_block_header()
- datum = self.datum_reader.read(self.datum_decoder)
+ datum = self.datum_reader.read(self.datum_decoder)
self.block_count -= 1
return datum
@@ -370,8 +399,6 @@ class DataFileReader(object):
def generate_sixteen_random_bytes():
try:
- import os
return os.urandom(16)
- except:
- import random
- return [ chr(random.randrange(256)) for i in range(16) ]
+ except NotImplementedError:
+ return [chr(random.randrange(256)) for i in range(16)]
diff --git a/lang/py/src/avro/protocol.py b/lang/py/src/avro/protocol.py
index f5c333b..117401c 100644
--- a/lang/py/src/avro/protocol.py
+++ b/lang/py/src/avro/protocol.py
@@ -1,3 +1,6 @@
+#!/usr/bin/env python
+
+##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -14,18 +17,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-"""
-Protocol implementation.
-"""
+"""Protocol implementation."""
-try:
- from hashlib import md5
-except ImportError:
- from md5 import md5
+from __future__ import absolute_import, division, print_function
+import hashlib
import json
-from avro import schema
+import avro.schema
#
# Constants
@@ -38,7 +37,7 @@ VALID_TYPE_SCHEMA_TYPES = ('enum', 'record', 'error', 'fixed')
# Exceptions
#
-class ProtocolParseException(schema.AvroException):
+class ProtocolParseException(avro.schema.AvroException):
pass
#
@@ -50,7 +49,7 @@ class Protocol(object):
def _parse_types(self, types, type_names):
type_objects = []
for type in types:
- type_object = schema.make_avsc_object(type, type_names)
+ type_object = avro.schema.make_avsc_object(type, type_names)
if type_object.type not in VALID_TYPE_SCHEMA_TYPES:
fail_msg = 'Type %s not an enum, fixed, record, or error.' % type
raise ProtocolParseException(fail_msg)
@@ -60,7 +59,7 @@ class Protocol(object):
def _parse_messages(self, messages, names):
message_objects = {}
for name, body in messages.iteritems():
- if message_objects.has_key(name):
+ if name in message_objects:
fail_msg = 'Message name "%s" repeated.' % name
raise ProtocolParseException(fail_msg)
try:
@@ -93,7 +92,7 @@ class Protocol(object):
self._props = {}
self.set_prop('name', name)
- type_names = schema.Names()
+ type_names = avro.schema.Names()
if namespace is not None:
self.set_prop('namespace', namespace)
type_names.default_namespace = namespace
@@ -101,13 +100,13 @@ class Protocol(object):
self.set_prop('types', self._parse_types(types, type_names))
if messages is not None:
self.set_prop('messages', self._parse_messages(messages, type_names))
- self._md5 = md5(str(self)).digest()
+ self._md5 = hashlib.md5(str(self)).digest()
# read-only properties
name = property(lambda self: self.get_prop('name'))
namespace = property(lambda self: self.get_prop('namespace'))
fullname = property(lambda self:
- schema.Name(self.name, self.namespace).fullname)
+ avro.schema.Name(self.name, self.namespace).fullname)
types = property(lambda self: self.get_prop('types'))
types_dict = property(lambda self: dict([(type.name, type)
for type in self.types]))
@@ -124,7 +123,7 @@ class Protocol(object):
def to_json(self):
to_dump = {}
to_dump['protocol'] = self.name
- names = schema.Names(default_namespace=self.namespace)
+ names = avro.schema.Names(default_namespace=self.namespace)
if self.namespace:
to_dump['namespace'] = self.namespace
if self.types:
@@ -149,20 +148,20 @@ class Message(object):
if not isinstance(request, list):
fail_msg = 'Request property not a list: %s' % request
raise ProtocolParseException(fail_msg)
- return schema.RecordSchema(None, None, request, names, 'request')
+ return avro.schema.RecordSchema(None, None, request, names, 'request')
def _parse_response(self, response, names):
if isinstance(response, basestring) and names.has_name(response, None):
return names.get_name(response, None)
else:
- return schema.make_avsc_object(response, names)
+ return avro.schema.make_avsc_object(response, names)
def _parse_errors(self, errors, names):
if not isinstance(errors, list):
fail_msg = 'Errors property not a list: %s' % errors
raise ProtocolParseException(fail_msg)
errors_for_parsing = {'type': 'error_union', 'declared_errors': errors}
- return schema.make_avsc_object(errors_for_parsing, names)
+ return avro.schema.make_avsc_object(errors_for_parsing, names)
def __init__(self, name, request, response, errors=None, names=None):
self._name = name
@@ -191,7 +190,7 @@ class Message(object):
def to_json(self, names=None):
if names is None:
- names = schema.Names()
+ names = avro.schema.Names()
to_dump = {}
to_dump['request'] = self.request.to_json(names)
to_dump['response'] = self.response.to_json(names)
@@ -217,7 +216,7 @@ def parse(json_string):
"""Constructs the Protocol from the JSON text."""
try:
json_data = json.loads(json_string)
- except:
+ except ValueError:
raise ProtocolParseException('Error parsing JSON: %s' % json_string)
# construct the Avro Protocol object
diff --git a/lang/py/src/avro/schema.py b/lang/py/src/avro/schema.py
index e694d7e..776d6e1 100644
--- a/lang/py/src/avro/schema.py
+++ b/lang/py/src/avro/schema.py
@@ -610,9 +610,10 @@ class MapSchema(Schema):
else:
try:
values_schema = make_avsc_object(values, names)
- except:
- fail_msg = 'Values schema not a valid Avro schema.'
- raise SchemaParseException(fail_msg)
+ except SchemaParseException:
+ raise
+ except Exception:
+ raise SchemaParseException('Values schema is not a valid Avro schema.')
self.set_prop('values', values_schema)
diff --git a/lang/py/test/sample_http_client.py b/lang/py/test/sample_http_client.py
index 4ad4925..62c91fd 100644
--- a/lang/py/test/sample_http_client.py
+++ b/lang/py/test/sample_http_client.py
@@ -72,7 +72,7 @@ if __name__ == '__main__':
try:
num_messages = int(sys.argv[4])
- except:
+ except IndexError:
num_messages = 1
# build the parameters for the request
diff --git a/lang/py/test/test_protocol.py b/lang/py/test/test_protocol.py
index 3a8649f..4d8e783 100644
--- a/lang/py/test/test_protocol.py
+++ b/lang/py/test/test_protocol.py
@@ -1,3 +1,6 @@
+#!/usr/bin/env python
+
+##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -5,17 +8,19 @@
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
-#
+#
# https://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-"""
-Test the protocol parsing logic.
-"""
+
+"""Test the protocol parsing logic."""
+
+from __future__ import absolute_import, division, print_function
+
import unittest
from avro import protocol
@@ -154,7 +159,7 @@ ExampleProtocol("""\
"types": [
{"name": "org.apache.avro.test.util.MD5", "type": "fixed", "size": 16},
- {"name": "ReferencedRecord", "type": "record",
+ {"name": "ReferencedRecord", "type": "record",
"fields": [ {"name": "foo", "type": "string"} ] },
{"name": "TestRecord", "type": "record",
"fields": [ {"name": "hash", "type": "org.apache.avro.test.util.MD5"},
@@ -167,7 +172,7 @@ ExampleProtocol("""\
"messages": {
"echo": {
- "request": [{"name": "qualified",
+ "request": [{"name": "qualified",
"type": "org.apache.avro.test.namespace.TestRecord"}],
"response": "TestRecord"
},
@@ -188,13 +193,13 @@ ExampleProtocol("""\
"types": [
{"name": "org.apache.avro.test.util.MD5", "type": "fixed", "size": 16},
- {"name": "ReferencedRecord", "type": "record",
- "namespace": "org.apache.avro.other.namespace",
+ {"name": "ReferencedRecord", "type": "record",
+ "namespace": "org.apache.avro.other.namespace",
"fields": [ {"name": "foo", "type": "string"} ] },
{"name": "TestRecord", "type": "record",
"fields": [ {"name": "hash", "type": "org.apache.avro.test.util.MD5"},
- {"name": "qualified",
- "type": "org.apache.avro.other.namespace.ReferencedRecord"}
+ {"name": "qualified",
+ "type": "org.apache.avro.other.namespace.ReferencedRecord"}
]
},
{"name": "TestError",
@@ -204,7 +209,7 @@ ExampleProtocol("""\
"messages": {
"echo": {
- "request": [{"name": "qualified",
+ "request": [{"name": "qualified",
"type": "org.apache.avro.test.namespace.TestRecord"}],
"response": "TestRecord"
},
@@ -225,10 +230,10 @@ ExampleProtocol("""\
"types": [
{"name": "org.apache.avro.test.util.MD5", "type": "fixed", "size": 16},
- {"name": "ReferencedRecord", "type": "record",
- "namespace": "org.apache.avro.other.namespace",
+ {"name": "ReferencedRecord", "type": "record",
+ "namespace": "org.apache.avro.other.namespace",
"fields": [ {"name": "foo", "type": "string"} ] },
- {"name": "ReferencedRecord", "type": "record",
+ {"name": "ReferencedRecord", "type": "record",
"fields": [ {"name": "bar", "type": "double"} ] },
{"name": "TestError",
"type": "error", "fields": [ {"name": "message", "type": "string"} ]
@@ -237,7 +242,7 @@ ExampleProtocol("""\
"messages": {
"echo": {
- "request": [{"name": "qualified",
+ "request": [{"name": "qualified",
"type": "ReferencedRecord"}],
"response": "org.apache.avro.other.namespace.ReferencedRecord"
},
@@ -258,9 +263,9 @@ ExampleProtocol("""\
"types": [
{"name": "org.apache.avro.test.util.MD5", "type": "fixed", "size": 16},
- {"name": "ReferencedRecord", "type": "record",
+ {"name": "ReferencedRecord", "type": "record",
"fields": [ {"name": "foo", "type": "string"} ] },
- {"name": "ReferencedRecord", "type": "record",
+ {"name": "ReferencedRecord", "type": "record",
"fields": [ {"name": "bar", "type": "double"} ] },
{"name": "TestError",
"type": "error", "fields": [ {"name": "message", "type": "string"} ]
@@ -269,7 +274,7 @@ ExampleProtocol("""\
"messages": {
"echo": {
- "request": [{"name": "qualified",
+ "request": [{"name": "qualified",
"type": "ReferencedRecord"}],
"response": "org.apache.avro.other.namespace.ReferencedRecord"
},
@@ -352,12 +357,12 @@ class TestProtocol(unittest.TestCase):
for example in EXAMPLES:
try:
protocol.parse(example.protocol_string)
- if example.valid:
+ if example.valid:
num_correct += 1
else:
self.fail("Parsed invalid protocol: %s" % (example.name,))
- except Exception, e:
- if not example.valid:
+ except Exception as e:
+ if not example.valid:
num_correct += 1
else:
self.fail("Coudl not parse valid protocol: %s" % (example.name,))
@@ -367,10 +372,10 @@ class TestProtocol(unittest.TestCase):
self.assertEqual(num_correct, len(EXAMPLES), fail_msg)
def test_inner_namespace_set(self):
- print ''
- print 'TEST INNER NAMESPACE'
- print '==================='
- print ''
+ print('')
+ print('TEST INNER NAMESPACE')
+ print('===================')
+ print('')
proto = protocol.parse(HELLO_WORLD.protocol_string)
self.assertEqual(proto.namespace, "com.acme")
greeting_type = proto.types_dict['Greeting']
@@ -388,23 +393,22 @@ class TestProtocol(unittest.TestCase):
Test that the string generated by an Avro Protocol object
is, in fact, a valid Avro protocol.
"""
- print ''
- print 'TEST CAST TO STRING'
- print '==================='
- print ''
+ print('')
+ print('TEST CAST TO STRING')
+ print('===================')
+ print('')
num_correct = 0
for example in VALID_EXAMPLES:
protocol_data = protocol.parse(example.protocol_string)
try:
- try:
- protocol.parse(str(protocol_data))
- debug_msg = "%s: STRING CAST SUCCESS" % example.name
- num_correct += 1
- except:
- debug_msg = "%s: STRING CAST FAILURE" % example.name
- finally:
- print debug_msg
+ protocol.parse(str(protocol_data))
+ except (ValueError, ProtocolParseException):
+ debug_msg = "%s: STRING CAST FAILURE" % example.name
+ else:
+ debug_msg = "%s: STRING CAST SUCCESS" % example.name
+ num_correct += 1
+ print(debug_msg)
fail_msg = "Cast to string success on %d out of %d protocols" % \
(num_correct, len(VALID_EXAMPLES))
@@ -417,10 +421,10 @@ class TestProtocol(unittest.TestCase):
to generate Avro protocol "round trip".
3. Ensure "original" and "round trip" protocols are equivalent.
"""
- print ''
- print 'TEST ROUND TRIP'
- print '==============='
- print ''
+ print('')
+ print('TEST ROUND TRIP')
+ print('===============')
+ print('')
num_correct = 0
for example in VALID_EXAMPLES:
@@ -430,7 +434,7 @@ class TestProtocol(unittest.TestCase):
if original_protocol == round_trip_protocol:
num_correct += 1
debug_msg = "%s: ROUND TRIP SUCCESS" % example.name
- else:
+ else:
self.fail("Round trip failure: %s %s %s", (example.name, example.protocol_string, str(original_protocol)))
fail_msg = "Round trip success on %d out of %d protocols" % \
diff --git a/lang/py/test/txsample_http_client.py b/lang/py/test/txsample_http_client.py
index 3841062..dba4ade 100644
--- a/lang/py/test/txsample_http_client.py
+++ b/lang/py/test/txsample_http_client.py
@@ -74,7 +74,7 @@ if __name__ == '__main__':
try:
num_messages = int(sys.argv[4])
- except:
+ except IndexError:
num_messages = 1
# build the parameters for the request
diff --git a/lang/py3/avro/tests/sample_http_client.py b/lang/py3/avro/tests/sample_http_client.py
index da8d030..47212a1 100644
--- a/lang/py3/avro/tests/sample_http_client.py
+++ b/lang/py3/avro/tests/sample_http_client.py
@@ -74,7 +74,7 @@ if __name__ == '__main__':
try:
num_messages = int(sys.argv[4])
- except:
+ except IndexError:
num_messages = 1
# build the parameters for the request
diff --git a/lang/py3/avro/tests/test_protocol.py b/lang/py3/avro/tests/test_protocol.py
index abd17e8..7dabd59 100644
--- a/lang/py3/avro/tests/test_protocol.py
+++ b/lang/py3/avro/tests/test_protocol.py
@@ -453,14 +453,13 @@ class TestProtocol(unittest.TestCase):
proto = protocol.Parse(example.protocol_string)
try:
protocol.Parse(str(proto))
- logging.debug(
- 'Successfully reparsed protocol:\n%s',
- example.protocol_string)
- num_correct += 1
- except:
- logging.debug(
- 'Failed to reparse protocol:\n%s',
- example.protocol_string)
+ except ProtocolParseException:
+ logging.debug('Failed to reparse protocol:\n%s',
+ example.protocol_string)
+ continue
+ logging.debug('Successfully reparsed protocol:\n%s',
+ example.protocol_string)
+ num_correct += 1
fail_msg = (
'Cast to string success on %d out of %d protocols'
diff --git a/lang/py3/avro/tests/txsample_http_client.py b/lang/py3/avro/tests/txsample_http_client.py
index fede960..f3e573a 100644
--- a/lang/py3/avro/tests/txsample_http_client.py
+++ b/lang/py3/avro/tests/txsample_http_client.py
@@ -76,7 +76,7 @@ if __name__ == '__main__':
try:
num_messages = int(sys.argv[4])
- except:
+ except IndexError:
num_messages = 1
# build the parameters for the request
diff --git a/lang/py3/setup.cfg b/lang/py3/setup.cfg
index 89b17fe..28fcd45 100644
--- a/lang/py3/setup.cfg
+++ b/lang/py3/setup.cfg
@@ -72,6 +72,6 @@ known_third_party=zope
[pycodestyle]
exclude = .eggs,build
-ignore = E111,E114,E121,E122,E124,E127,E128,E129,E201,E202,E203,E221,E225,E231,E241,E261,E301,E302,E303,E305,E402,E701,E703,E722,W503,W504
+ignore = E111,E114,E121,E122,E124,E127,E128,E129,E201,E202,E203,E221,E225,E231,E241,E261,E301,E302,E303,E305,E402,E701,E703,W503,W504
max-line-length = 150
statistics = True
[avro] 07/23: AVRO-2445: Remove StoppableHTTPServer Polyfill (#567)
Posted by rs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rskraba pushed a commit to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/avro.git
commit 62e2db9866f1acdbf08e102d7b2ec732ef358918
Author: Michael A. Smith <mi...@syapse.com>
AuthorDate: Tue Jul 23 05:44:29 2019 -0400
AVRO-2445: Remove StoppableHTTPServer Polyfill (#567)
---
lang/py/src/avro/tool.py | 34 ++++++++++------------------------
1 file changed, 10 insertions(+), 24 deletions(-)
diff --git a/lang/py/src/avro/tool.py b/lang/py/src/avro/tool.py
index be2ef41..6a92fee 100644
--- a/lang/py/src/avro/tool.py
+++ b/lang/py/src/avro/tool.py
@@ -19,13 +19,14 @@ Command-line tool
NOTE: The API for the command-line tool is experimental.
"""
+
import sys
-from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
+import threading
import urlparse
-from avro import io
-from avro import datafile
-from avro import protocol
-from avro import ipc
+from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
+
+from avro import datafile, io, ipc, protocol
+
class GenericResponder(ipc.Responder):
def __init__(self, proto, msg, datum):
@@ -55,24 +56,9 @@ class GenericHandler(BaseHTTPRequestHandler):
resp_writer.write_framed_message(resp_body)
if server_should_shutdown:
print >> sys.stderr, "Shutting down server."
- self.server.force_stop()
-
-class StoppableHTTPServer(HTTPServer):
- """HTTPServer.shutdown added in Python 2.6. FML."""
- stopped = False
- allow_reuse_address = True
- def __init__(self, *args, **kw):
- HTTPServer.__init__(self, *args, **kw)
- self.allow_reuse_address = True
-
- def serve_forever(self):
- while not self.stopped:
- self.handle_request()
-
- def force_stop(self):
- self.server_close()
- self.stopped = True
- self.serve_forever()
+ quitter = threading.Thread(target=self.server.shutdown)
+ quitter.daemon = True
+ quitter.start()
def run_server(uri, proto, msg, datum):
url_obj = urlparse.urlparse(uri)
@@ -81,7 +67,7 @@ def run_server(uri, proto, msg, datum):
global server_should_shutdown
server_should_shutdown = False
responder = GenericResponder(proto, msg, datum)
- server = StoppableHTTPServer(server_addr, GenericHandler)
+ server = HTTPServer(server_addr, GenericHandler)
print "Port: %s" % server.server_port
sys.stdout.flush()
server.allow_reuse_address = True
[avro] 09/23: AVRO-2410: Framework for Linting (#569)
Posted by rs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rskraba pushed a commit to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/avro.git
commit 89ed6727738b829c7c0903efe6693c432044c1bc
Author: Michael A. Smith <mi...@syapse.com>
AuthorDate: Thu Jul 18 07:02:23 2019 -0400
AVRO-2410: Framework for Linting (#569)
* AVRO-2410: build.sh Framework for Linting
* AVRO-2410: Sort Python Imports with isort
* AVRO-2410: Sort Python3 Imports with isort
---
.editorconfig | 6 +--
build.sh | 12 +++--
lang/c++/build.sh | 6 ++-
lang/c/build.sh | 7 ++-
lang/csharp/build.sh | 8 +--
lang/java/build.sh | 70 +++++++++++----------------
lang/java/checkstyle.xml | 2 -
lang/js/build.sh | 7 +--
lang/perl/build.sh | 5 +-
lang/php/build.sh | 6 ++-
lang/py/.gitignore | 1 +
lang/py/build.sh | 67 +++++++++++--------------
lang/{py3/build.sh => py/setup.cfg} | 34 ++++---------
lang/py/src/avro/datafile.py | 5 +-
lang/py/src/avro/io.py | 12 ++---
lang/py/src/avro/ipc.py | 7 ++-
lang/py/src/avro/schema.py | 4 +-
lang/py/src/avro/tether/tether_task.py | 14 +++---
lang/py/src/avro/tether/tether_task_runner.py | 10 ++--
lang/py/src/avro/tether/util.py | 2 +-
lang/py/src/avro/timezones.py | 4 +-
lang/py/src/avro/txipc.py | 11 ++---
lang/py/test/av_bench.py | 5 +-
lang/py/test/gen_interop_data.py | 5 +-
lang/py/test/mock_tether_parent.py | 9 ++--
lang/py/test/sample_http_client.py | 3 +-
lang/py/test/sample_http_server.py | 5 +-
lang/py/test/set_avro_test_path.py | 4 +-
lang/py/test/test_datafile.py | 5 +-
lang/py/test/test_datafile_interop.py | 3 +-
lang/py/test/test_io.py | 12 ++---
lang/py/test/test_ipc.py | 2 +-
lang/py/test/test_protocol.py | 2 +
lang/py/test/test_schema.py | 5 +-
lang/py/test/test_script.py | 31 +++---------
lang/py/test/test_tether_task.py | 3 +-
lang/py/test/test_tether_word_count.py | 3 +-
lang/py/test/txsample_http_client.py | 6 +--
lang/py/test/txsample_http_server.py | 8 ++-
lang/py/test/word_count_task.py | 3 +-
lang/py3/avro/datafile.py | 2 +-
lang/py3/avro/ipc.py | 3 +-
lang/py3/avro/protocol.py | 3 +-
lang/py3/avro/schema.py | 3 +-
lang/py3/avro/schemanormalization.py | 2 +-
lang/py3/avro/tests/av_bench.py | 1 -
lang/py3/avro/tests/gen_interop_data.py | 5 +-
lang/py3/avro/tests/run_tests.py | 2 +-
lang/py3/avro/tests/sample_http_client.py | 3 +-
lang/py3/avro/tests/sample_http_server.py | 5 +-
lang/py3/avro/tests/test_datafile.py | 14 ++++--
lang/py3/avro/tests/test_enum.py | 1 +
lang/py3/avro/tests/test_io.py | 1 -
lang/py3/avro/tests/test_ipc.py | 4 +-
lang/py3/avro/tests/test_normalization.py | 2 +-
lang/py3/avro/tests/test_protocol.py | 1 -
lang/py3/avro/tests/test_schema.py | 1 -
lang/py3/avro/tests/test_script.py | 1 -
lang/py3/avro/tests/txsample_http_client.py | 6 +--
lang/py3/avro/tests/txsample_http_server.py | 9 ++--
lang/py3/avro/tool.py | 7 +--
lang/py3/avro/txipc.py | 7 ++-
lang/py3/build.sh | 7 +--
lang/py3/setup.cfg | 9 ++++
lang/py3/setup.py | 52 ++++++++++++++++++--
lang/ruby/build.sh | 39 +++++++--------
share/docker/Dockerfile | 8 +++
share/docker/run-tests.sh | 2 +-
68 files changed, 310 insertions(+), 304 deletions(-)
diff --git a/.editorconfig b/.editorconfig
index f3b4fdc..6e7532c 100644
--- a/.editorconfig
+++ b/.editorconfig
@@ -34,6 +34,6 @@ trim_trailing_whitespace=true
#indent_style = space
#indent_size = 2
-#[*.py]
-#indent_style = space
-#indent_size = 4
+[*.py]
+indent_style = space
+indent_size = 2
diff --git a/build.sh b/build.sh
index 6158ce8..ab7ccd8 100755
--- a/build.sh
+++ b/build.sh
@@ -23,7 +23,7 @@ VERSION=`cat share/VERSION.txt`
DOCKER_XTRA_ARGS=""
function usage {
- echo "Usage: $0 {test|dist|sign|clean|veryclean|docker [--args \"docker-args\"]|rat|githooks|docker-test}"
+ echo "Usage: $0 {lint|test|dist|sign|clean|veryclean|docker [--args \"docker-args\"]|rat|githooks|docker-test}"
exit 1
}
@@ -40,6 +40,12 @@ do
shift
case "$target" in
+ lint)
+ for lang_dir in lang/*; do
+ (cd "$lang_dir" && ./build.sh lint)
+ done
+ ;;
+
test)
# run lang-specific tests
(cd lang/java; ./build.sh test)
@@ -50,8 +56,8 @@ do
# install java artifacts required by other builds and interop tests
mvn -B install -DskipTests
- (cd lang/py; ./build.sh test)
- (cd lang/py3; ./build.sh test)
+ (cd lang/py && ./build.sh lint test)
+ (cd lang/py3 && ./build.sh lint test)
(cd lang/c; ./build.sh test)
(cd lang/c++; ./build.sh test)
(cd lang/csharp; ./build.sh test)
diff --git a/lang/c++/build.sh b/lang/c++/build.sh
index bd1a0b8..543638b 100755
--- a/lang/c++/build.sh
+++ b/lang/c++/build.sh
@@ -18,7 +18,7 @@
set -e # exit on error
function usage {
- echo "Usage: $0 {test|dist|clean|install|doc}"
+ echo "Usage: $0 {lint|test|dist|clean|install|doc}"
exit 1
}
@@ -76,6 +76,10 @@ for target in "$@"
do
case "$target" in
+ lint)
+ echo 'This is a stub where someone can provide linting.'
+ ;;
+
test)
(cd build && cmake -G "Unix Makefiles" -D CMAKE_BUILD_TYPE=Debug -D AVRO_ADD_PROTECTOR_FLAGS=1 .. && make && cd .. \
&& ./build/buffertest \
diff --git a/lang/c/build.sh b/lang/c/build.sh
index a813b64..ff0c16f 100755
--- a/lang/c/build.sh
+++ b/lang/c/build.sh
@@ -19,7 +19,6 @@
#
set -e # exit on error
-#set -x
root_dir=$(pwd)
build_dir="../../build/c"
@@ -57,6 +56,10 @@ case "$1" in
$build_dir/tests/test_interop_data "../../build/interop/data"
;;
+ lint)
+ echo 'This is a stub where someone can provide linting.'
+ ;;
+
test)
prepare_build
make -C $build_dir
@@ -88,7 +91,7 @@ case "$1" in
;;
*)
- echo "Usage: $0 {interop-data-generate|interop-data-test|test|dist|clean}"
+ echo "Usage: $0 {interop-data-generate|interop-data-test|lint|test|dist|clean}"
exit 1
esac
diff --git a/lang/csharp/build.sh b/lang/csharp/build.sh
index db344e2..afa4859 100755
--- a/lang/csharp/build.sh
+++ b/lang/csharp/build.sh
@@ -25,6 +25,10 @@ VERSION=`cat $ROOT/share/VERSION.txt`
case "$1" in
+ lint)
+ echo 'This is a stub where someone can provide linting.'
+ ;;
+
test)
dotnet build --configuration Release Avro.sln
@@ -77,8 +81,6 @@ case "$1" in
;;
*)
- echo "Usage: $0 {test|clean|dist|perf|interop-data-generate|interop-data-test}"
+ echo "Usage: $0 {lint|test|clean|dist|perf|interop-data-generate|interop-data-test}"
exit 1
esac
-
-exit 0
diff --git a/lang/java/build.sh b/lang/java/build.sh
index f0f0f32..88847ea 100755
--- a/lang/java/build.sh
+++ b/lang/java/build.sh
@@ -15,51 +15,37 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-set -e # exit on error
+set -e
-function usage {
- echo "Usage: $0 {test|dist|clean}"
+usage() {
+ echo "Usage: $0 {lint|test|dist|clean}"
exit 1
}
-if [ $# -eq 0 ]
-then
- usage
-fi
-
-if [ -f VERSION.txt ]
-then
- VERSION=`cat VERSION.txt`
-else
- VERSION=`cat ../../share/VERSION.txt`
-fi
-
-for target in "$@"
-do
-
-function do_dist() {
- mvn -P dist package -DskipTests javadoc:aggregate
+main() {
+ local target
+ (( $# )) || usage
+ for target; do
+ case "$target" in
+ lint)
+ mvn -B spotless:apply
+ ;;
+ test)
+ mvn -B test
+ # Test the modules that depend on hadoop using Hadoop 3
+ mvn -B test -Phadoop3
+ ;;
+ dist)
+ mvn -P dist package -DskipTests javadoc:aggregate
+ ;;
+ clean)
+ mvn clean
+ ;;
+ *)
+ usage
+ ;;
+ esac
+ done
}
-case "$target" in
- test)
- mvn -B test
- # Test the modules that depend on hadoop using Hadoop 3
- mvn -B test -Phadoop3
- ;;
-
- dist)
- do_dist
- ;;
-
- clean)
- mvn clean
- ;;
-
- *)
- usage
-esac
-
-done
-
-exit 0
+main "$@"
diff --git a/lang/java/checkstyle.xml b/lang/java/checkstyle.xml
index 0391815..e817cf8 100644
--- a/lang/java/checkstyle.xml
+++ b/lang/java/checkstyle.xml
@@ -27,8 +27,6 @@
<property name="file" value="${checkstyle.suppressions.file}"/>
</module>
- <module name="SuppressionCommentFilter"/>
-
<module name="FileTabCharacter"/>
<module name="NewlineAtEndOfFile">
<property name="lineSeparator" value="lf"/>
diff --git a/lang/js/build.sh b/lang/js/build.sh
index deafeb5..a6cdb6b 100755
--- a/lang/js/build.sh
+++ b/lang/js/build.sh
@@ -20,6 +20,9 @@ set -e
cd `dirname "$0"`
case "$1" in
+ lint)
+ echo 'This is a stub where someone can provide linting.'
+ ;;
test)
npm install
npm run cover
@@ -33,8 +36,6 @@ case "$1" in
rm -rf coverage
;;
*)
- echo "Usage: $0 {test|dist|clean}" >&2
+ echo "Usage: $0 {lint|test|dist|clean}" >&2
exit 1
esac
-
-exit 0
diff --git a/lang/perl/build.sh b/lang/perl/build.sh
index 95cc629..fee0ca6 100755
--- a/lang/perl/build.sh
+++ b/lang/perl/build.sh
@@ -18,7 +18,7 @@
set -e # exit on error
function usage {
- echo "Usage: $0 {test|dist|clean}"
+ echo "Usage: $0 {lint|test|dist|clean}"
exit 1
}
@@ -44,6 +44,9 @@ function do_clean(){
}
case "$target" in
+ lint)
+ echo 'This is a stub where someone can provide linting.'
+ ;;
test)
perl ./Makefile.PL && make test
;;
diff --git a/lang/php/build.sh b/lang/php/build.sh
index 46a568c..796c41e 100755
--- a/lang/php/build.sh
+++ b/lang/php/build.sh
@@ -53,6 +53,10 @@ case "$1" in
phpunit test/InterOpTest.php
;;
+ lint)
+ echo 'This is a stub where someone can provide linting.'
+ ;;
+
test)
phpunit test/AllTests.php
;;
@@ -66,7 +70,7 @@ case "$1" in
;;
*)
- echo "Usage: $0 {interop-data-generate|test-interop|test|dist|clean}"
+ echo "Usage: $0 {interop-data-generate|test-interop|lint|test|dist|clean}"
esac
diff --git a/lang/py/.gitignore b/lang/py/.gitignore
index df3cd4d..44d9b58 100644
--- a/lang/py/.gitignore
+++ b/lang/py/.gitignore
@@ -1,3 +1,4 @@
build/
lib/
userlogs/
+*.egg-info/
diff --git a/lang/py/build.sh b/lang/py/build.sh
index ff61989..662d854 100755
--- a/lang/py/build.sh
+++ b/lang/py/build.sh
@@ -15,46 +15,37 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-set -e # exit on error
+set -e
-function usage {
- echo "Usage: $0 {test|dist|clean}"
+usage() {
+ echo "Usage: $0 {lint|test|dist|clean}"
exit 1
}
-if [ $# -eq 0 ]
-then
- usage
-fi
-
-if [ -f VERSION.txt ]
-then
- VERSION=`cat VERSION.txt`
-else
- VERSION=`cat ../../share/VERSION.txt`
-fi
-
-for target in "$@"
-do
-
-case "$target" in
- test)
- ant test
- ;;
-
- dist)
- ant dist
- ;;
-
- clean)
- ant clean
- rm -rf userlogs/
- ;;
-
- *)
- usage
-esac
-
-done
+main() {
+ local target
+ (( $# )) || usage
+ for target; do
+ case "$target" in
+ lint)
+ ./setup.py isort
+ pycodestyle .
+ ;;
+ test)
+ ant test
+ ;;
+ dist)
+ ant dist
+ ;;
+ clean)
+ ant clean
+ rm -rf userlogs/
+ ;;
+ *)
+ usage
+ ;;
+ esac
+ done
+}
-exit 0
+main "$@"
diff --git a/lang/py3/build.sh b/lang/py/setup.cfg
old mode 100755
new mode 100644
similarity index 65%
copy from lang/py3/build.sh
copy to lang/py/setup.cfg
index 1bb27ad..dc07a1d
--- a/lang/py3/build.sh
+++ b/lang/py/setup.cfg
@@ -1,5 +1,4 @@
-#!/bin/bash
-
+##
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -7,7 +6,7 @@
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
-# https://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
@@ -15,25 +14,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-set -e # exit on error
-
-usage() {
- echo "Usage: $0 {test|dist|clean}"
-}
-
-main() {
- local target
- if (( $# == 0 )); then
- usage
- return 1
- fi
- for target; do
- case "$target" in
- clean|dist|test) : ;;
- *) usage; return 1 ;;
- esac
- done
- python3 setup.py "$@"
-}
+[isort]
+line_length = 150
+known_third_party=zope
-main "$@"
+[pycodestyle]
+max-line-length = 150
+statistics = True
+exclude = build
+ignore = E101,E111,E114,E121,E122,E124,E125,E126,E127,E128,E129,E201,E202,E203,E222,E226,E225,E231,E241,E251,E261,E262,E265,E266,E301,E302,E303,E305,E306,E402,E501,E701,E703,E704,E711,W191,W291,W292,W293,W391,W503,W601
diff --git a/lang/py/src/avro/datafile.py b/lang/py/src/avro/datafile.py
index c85a896..772d949 100644
--- a/lang/py/src/avro/datafile.py
+++ b/lang/py/src/avro/datafile.py
@@ -17,12 +17,13 @@
Read/Write Avro File Object Containers.
"""
import zlib
+
+from avro import io, schema
+
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
-from avro import schema
-from avro import io
try:
import snappy
has_snappy = True
diff --git a/lang/py/src/avro/io.py b/lang/py/src/avro/io.py
index d94f0ff..c36c5b3 100644
--- a/lang/py/src/avro/io.py
+++ b/lang/py/src/avro/io.py
@@ -38,18 +38,14 @@ uses the following mapping:
* Schema booleans are implemented as bool.
"""
+import datetime
+import json
import struct
-from avro import schema
-from avro import constants
-from avro import timezones
import sys
from binascii import crc32
-import datetime
+from decimal import Decimal, getcontext
-from decimal import Decimal
-from decimal import getcontext
-
-import json
+from avro import constants, schema, timezones
#
# Constants
diff --git a/lang/py/src/avro/ipc.py b/lang/py/src/avro/ipc.py
index c616ac0..8cbf07b 100644
--- a/lang/py/src/avro/ipc.py
+++ b/lang/py/src/avro/ipc.py
@@ -17,13 +17,13 @@
Support for inter-process calls.
"""
import httplib
+
+from avro import io, protocol, schema
+
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
-from avro import io
-from avro import protocol
-from avro import schema
#
# Constants
@@ -482,4 +482,3 @@ class HTTPTransceiver(object):
#
# Server Implementations (none yet)
#
-
diff --git a/lang/py/src/avro/schema.py b/lang/py/src/avro/schema.py
index 52684c2..e694d7e 100644
--- a/lang/py/src/avro/schema.py
+++ b/lang/py/src/avro/schema.py
@@ -35,10 +35,8 @@ A schema may be one of:
Null.
"""
-from math import floor
-from math import log10
-
import json
+from math import floor, log10
from avro import constants
diff --git a/lang/py/src/avro/tether/tether_task.py b/lang/py/src/avro/tether/tether_task.py
index 2588e1a..23112a7 100644
--- a/lang/py/src/avro/tether/tether_task.py
+++ b/lang/py/src/avro/tether/tether_task.py
@@ -18,19 +18,17 @@
__all__=["TetherTask","TaskType","inputProtocol","outputProtocol","HTTPRequestor"]
-from avro import schema, protocol
-from avro import io as avio
-from avro import ipc
-
+import collections
import io as pyio
-import sys
+import logging
import os
+import sys
+import threading
import traceback
-import logging
-import collections
from StringIO import StringIO
-import threading
+from avro import io as avio
+from avro import ipc, protocol, schema
# create protocol objects for the input and output protocols
# The build process should copy InputProtocol.avpr and OutputProtocol.avpr
diff --git a/lang/py/src/avro/tether/tether_task_runner.py b/lang/py/src/avro/tether/tether_task_runner.py
index 6cea49b..33d224a 100644
--- a/lang/py/src/avro/tether/tether_task_runner.py
+++ b/lang/py/src/avro/tether/tether_task_runner.py
@@ -26,13 +26,15 @@ if __name__ == "__main__":
else:
from . import TetherTask, find_port, inputProtocol
-from avro import ipc
-from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
import logging
-import weakref
-import threading
import sys
+import threading
import traceback
+import weakref
+from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
+
+from avro import ipc
+
class TaskRunnerResponder(ipc.Responder):
"""
diff --git a/lang/py/src/avro/tether/util.py b/lang/py/src/avro/tether/util.py
index e223db3..cbeeef0 100644
--- a/lang/py/src/avro/tether/util.py
+++ b/lang/py/src/avro/tether/util.py
@@ -31,4 +31,4 @@ def find_port():
port=s.getsockname()[1]
s.close()
- return port
\ No newline at end of file
+ return port
diff --git a/lang/py/src/avro/timezones.py b/lang/py/src/avro/timezones.py
index 7748f83..a4985b4 100644
--- a/lang/py/src/avro/timezones.py
+++ b/lang/py/src/avro/timezones.py
@@ -14,9 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from datetime import datetime
-from datetime import timedelta
-from datetime import tzinfo
+from datetime import datetime, timedelta, tzinfo
class UTCTzinfo(tzinfo):
diff --git a/lang/py/src/avro/txipc.py b/lang/py/src/avro/txipc.py
index a8572c6..72d63a6 100644
--- a/lang/py/src/avro/txipc.py
+++ b/lang/py/src/avro/txipc.py
@@ -19,17 +19,16 @@ try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
-from avro import ipc
-from avro import io
-
from zope.interface import implements
+from avro import io, ipc
+from twisted.internet.defer import Deferred, maybeDeferred
+from twisted.internet.protocol import Protocol
+from twisted.web import resource, server
from twisted.web.client import Agent
from twisted.web.http_headers import Headers
-from twisted.internet.defer import maybeDeferred, Deferred
from twisted.web.iweb import IBodyProducer
-from twisted.web import resource, server
-from twisted.internet.protocol import Protocol
+
class TwistedRequestor(ipc.BaseRequestor):
"""A Twisted-compatible requestor. Returns a Deferred that will fire with the
diff --git a/lang/py/test/av_bench.py b/lang/py/test/av_bench.py
index 03ab8ad..e90c987 100644
--- a/lang/py/test/av_bench.py
+++ b/lang/py/test/av_bench.py
@@ -18,13 +18,12 @@
import sys
import time
-from random import sample, choice, randint
+from random import choice, randint, sample
from string import lowercase
import avro.datafile
-import avro.schema
import avro.io
-
+import avro.schema
types = ["A", "CNAME"]
diff --git a/lang/py/test/gen_interop_data.py b/lang/py/test/gen_interop_data.py
index d8cd857..336434e 100644
--- a/lang/py/test/gen_interop_data.py
+++ b/lang/py/test/gen_interop_data.py
@@ -16,9 +16,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
-from avro import schema
-from avro import io
-from avro import datafile
+
+from avro import datafile, io, schema
DATUM = {
'intField': 12,
diff --git a/lang/py/test/mock_tether_parent.py b/lang/py/test/mock_tether_parent.py
index 99f4153..c82e249 100644
--- a/lang/py/test/mock_tether_parent.py
+++ b/lang/py/test/mock_tether_parent.py
@@ -14,14 +14,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import socket
import sys
-import set_avro_test_path
from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
-from avro import ipc
-from avro import protocol
-from avro import tether
-import socket
+import set_avro_test_path
+from avro import ipc, protocol, tether
+
def find_port():
"""
diff --git a/lang/py/test/sample_http_client.py b/lang/py/test/sample_http_client.py
index 1ae29e4..4ad4925 100644
--- a/lang/py/test/sample_http_client.py
+++ b/lang/py/test/sample_http_client.py
@@ -17,8 +17,7 @@
# limitations under the License.
import sys
-from avro import ipc
-from avro import protocol
+from avro import ipc, protocol
MAIL_PROTOCOL_JSON = """\
{"namespace": "example.proto",
diff --git a/lang/py/test/sample_http_server.py b/lang/py/test/sample_http_server.py
index 9f05e94..e412ab5 100644
--- a/lang/py/test/sample_http_server.py
+++ b/lang/py/test/sample_http_server.py
@@ -15,9 +15,10 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
-from avro import ipc
-from avro import protocol
+
+from avro import ipc, protocol
MAIL_PROTOCOL_JSON = """\
{"namespace": "example.proto",
diff --git a/lang/py/test/set_avro_test_path.py b/lang/py/test/set_avro_test_path.py
index 8c9e242..8e47faf 100644
--- a/lang/py/test/set_avro_test_path.py
+++ b/lang/py/test/set_avro_test_path.py
@@ -28,8 +28,8 @@ being built. To work around this the unittests import this module before
importing AVRO. This module in turn adjusts the python path so that the test
build of AVRO is higher on the path then any installed eggs.
"""
-import sys
import os
+import sys
# determine the build directory and then make sure all paths that start with the
# build directory are at the top of the path
@@ -37,4 +37,4 @@ builddir=os.path.split(os.path.split(__file__)[0])[0]
bpaths=filter(lambda s:s.startswith(builddir), sys.path)
for p in bpaths:
- sys.path.insert(0,p)
\ No newline at end of file
+ sys.path.insert(0,p)
diff --git a/lang/py/test/test_datafile.py b/lang/py/test/test_datafile.py
index 102d9b0..2b7061c 100644
--- a/lang/py/test/test_datafile.py
+++ b/lang/py/test/test_datafile.py
@@ -17,10 +17,7 @@ import os
import unittest
import set_avro_test_path
-
-from avro import schema
-from avro import io
-from avro import datafile
+from avro import datafile, io, schema
SCHEMAS_TO_VALIDATE = (
('"null"', None),
diff --git a/lang/py/test/test_datafile_interop.py b/lang/py/test/test_datafile_interop.py
index 545ce6c..ee02f99 100644
--- a/lang/py/test/test_datafile_interop.py
+++ b/lang/py/test/test_datafile_interop.py
@@ -17,9 +17,8 @@ import os
import unittest
import set_avro_test_path
+from avro import datafile, io
-from avro import io
-from avro import datafile
class TestDataFileInterop(unittest.TestCase):
def test_interop(self):
diff --git a/lang/py/test/test_io.py b/lang/py/test/test_io.py
index 8ba0092..533aa40 100644
--- a/lang/py/test/test_io.py
+++ b/lang/py/test/test_io.py
@@ -13,22 +13,20 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+import datetime
import unittest
-
+from binascii import hexlify
from decimal import Decimal
+import set_avro_test_path
+from avro import io, schema, timezones
+
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
-from binascii import hexlify
-import datetime
-import set_avro_test_path
-from avro import schema
-from avro import io
-from avro import timezones
SCHEMAS_TO_VALIDATE = (
('"null"', None),
diff --git a/lang/py/test/test_ipc.py b/lang/py/test/test_ipc.py
index 97d280e..575a0c9 100644
--- a/lang/py/test/test_ipc.py
+++ b/lang/py/test/test_ipc.py
@@ -20,11 +20,11 @@ servers yet available.
import unittest
import set_avro_test_path
-
# This test does import this code, to make sure it at least passes
# compilation.
from avro import ipc
+
class TestIPC(unittest.TestCase):
def test_placeholder(self):
pass
diff --git a/lang/py/test/test_protocol.py b/lang/py/test/test_protocol.py
index 483dfcf..3a8649f 100644
--- a/lang/py/test/test_protocol.py
+++ b/lang/py/test/test_protocol.py
@@ -17,8 +17,10 @@
Test the protocol parsing logic.
"""
import unittest
+
from avro import protocol
+
class ExampleProtocol(object):
def __init__(self, protocol_string, valid, name='', comment=''):
self._protocol_string = protocol_string
diff --git a/lang/py/test/test_schema.py b/lang/py/test/test_schema.py
index a4ae4cf..dd5aa8a 100644
--- a/lang/py/test/test_schema.py
+++ b/lang/py/test/test_schema.py
@@ -18,11 +18,10 @@ Test the schema parsing logic.
"""
import unittest
-from avro.schema import SchemaParseException, AvroException
-
import set_avro_test_path
-
from avro import schema
+from avro.schema import AvroException, SchemaParseException
+
def print_test_name(test_name):
print ''
diff --git a/lang/py/test/test_script.py b/lang/py/test/test_script.py
index df02a98..214fc15 100644
--- a/lang/py/test/test_script.py
+++ b/lang/py/test/test_script.py
@@ -14,37 +14,22 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import unittest
import csv
-from cStringIO import StringIO
import json
+import unittest
+from cStringIO import StringIO
+from operator import itemgetter
+from os import remove
+from os.path import dirname, isfile, join
+from subprocess import check_call, check_output
from tempfile import NamedTemporaryFile
+
import avro.schema
-from avro.io import DatumWriter
from avro.datafile import DataFileWriter
-from os.path import dirname, join, isfile
-from os import remove
-from operator import itemgetter
+from avro.io import DatumWriter
NUM_RECORDS = 7
-try:
- from subprocess import check_output
-except ImportError:
- from subprocess import Popen, PIPE
-
- def check_output(args):
- pipe = Popen(args, stdout=PIPE)
- if pipe.wait() != 0:
- raise ValueError
- return pipe.stdout.read()
-
-try:
- from subprocess import check_call
-except ImportError:
- def check_call(args, **kw):
- pipe = Popen(args, **kw)
- assert pipe.wait() == 0
SCHEMA = '''
{
diff --git a/lang/py/test/test_tether_task.py b/lang/py/test/test_tether_task.py
index ed87083..9933070 100644
--- a/lang/py/test/test_tether_task.py
+++ b/lang/py/test/test_tether_task.py
@@ -24,6 +24,7 @@ import unittest
import set_avro_test_path
+
class TestTetherTask(unittest.TestCase):
"""
TODO: We should validate the the server response by looking at stdout
@@ -113,4 +114,4 @@ class TestTetherTask(unittest.TestCase):
pass
if __name__ == '__main__':
- unittest.main()
\ No newline at end of file
+ unittest.main()
diff --git a/lang/py/test/test_tether_word_count.py b/lang/py/test/test_tether_word_count.py
index 3488462..cc34f2b 100644
--- a/lang/py/test/test_tether_word_count.py
+++ b/lang/py/test/test_tether_word_count.py
@@ -15,14 +15,15 @@
# limitations under the License.
import inspect
+import os
import subprocess
import sys
import time
import unittest
-import os
import set_avro_test_path
+
class TestTetherWordCount(unittest.TestCase):
""" unittest for a python tethered map-reduce job.
"""
diff --git a/lang/py/test/txsample_http_client.py b/lang/py/test/txsample_http_client.py
index 807e50f..3841062 100644
--- a/lang/py/test/txsample_http_client.py
+++ b/lang/py/test/txsample_http_client.py
@@ -17,12 +17,10 @@
# limitations under the License.
import sys
-from twisted.internet import reactor, defer
+from avro import protocol, txipc
+from twisted.internet import defer, reactor
from twisted.python.util import println
-from avro import protocol
-from avro import txipc
-
MAIL_PROTOCOL_JSON = """\
{"namespace": "example.proto",
"protocol": "Mail",
diff --git a/lang/py/test/txsample_http_server.py b/lang/py/test/txsample_http_server.py
index 36cdb9f..604ef54 100644
--- a/lang/py/test/txsample_http_server.py
+++ b/lang/py/test/txsample_http_server.py
@@ -15,12 +15,10 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-from twisted.web import server
-from twisted.internet import reactor
-from avro import ipc
-from avro import protocol
-from avro import txipc
+from avro import ipc, protocol, txipc
+from twisted.internet import reactor
+from twisted.web import server
MAIL_PROTOCOL_JSON = """\
{"namespace": "example.proto",
diff --git a/lang/py/test/word_count_task.py b/lang/py/test/word_count_task.py
index ad911ad..24f2da2 100644
--- a/lang/py/test/word_count_task.py
+++ b/lang/py/test/word_count_task.py
@@ -18,9 +18,10 @@
__all__=["WordCountTask"]
+import logging
+
from avro.tether import TetherTask
-import logging
#TODO::Make the logging level a parameter we can set
#logging.basicConfig(level=logging.INFO)
diff --git a/lang/py3/avro/datafile.py b/lang/py3/avro/datafile.py
index adb2429..1e9b3fa 100644
--- a/lang/py3/avro/datafile.py
+++ b/lang/py3/avro/datafile.py
@@ -25,8 +25,8 @@ import logging
import os
import zlib
-from avro import schema
from avro import io as avro_io
+from avro import schema
try:
import snappy
diff --git a/lang/py3/avro/ipc.py b/lang/py3/avro/ipc.py
index 0bca4cb..f5d376d 100644
--- a/lang/py3/avro/ipc.py
+++ b/lang/py3/avro/ipc.py
@@ -29,8 +29,7 @@ import os
import socketserver
from avro import io as avro_io
-from avro import protocol
-from avro import schema
+from avro import protocol, schema
logger = logging.getLogger(__name__)
diff --git a/lang/py3/avro/protocol.py b/lang/py3/avro/protocol.py
index b40fa08..30834da 100644
--- a/lang/py3/avro/protocol.py
+++ b/lang/py3/avro/protocol.py
@@ -21,11 +21,10 @@
Protocol implementation.
"""
-from types import MappingProxyType
-
import hashlib
import json
import logging
+from types import MappingProxyType
from avro import schema
diff --git a/lang/py3/avro/schema.py b/lang/py3/avro/schema.py
index b0b6c4c..9fbe8e5 100644
--- a/lang/py3/avro/schema.py
+++ b/lang/py3/avro/schema.py
@@ -38,12 +38,11 @@ A schema may be one of:
- Null.
"""
-from types import MappingProxyType
-
import abc
import json
import logging
import re
+from types import MappingProxyType
logger = logging.getLogger(__name__)
diff --git a/lang/py3/avro/schemanormalization.py b/lang/py3/avro/schemanormalization.py
index 9e6d364..88cddba 100644
--- a/lang/py3/avro/schemanormalization.py
+++ b/lang/py3/avro/schemanormalization.py
@@ -21,7 +21,7 @@
import hashlib
from io import StringIO
-from avro.schema import (ERROR, RECORD, UNION, ARRAY, MAP, ENUM, FIXED)
+from avro.schema import ARRAY, ENUM, ERROR, FIXED, MAP, RECORD, UNION
def ToParsingCanonicalForm(schema):
diff --git a/lang/py3/avro/tests/av_bench.py b/lang/py3/avro/tests/av_bench.py
index 78561c4..88cbafb 100644
--- a/lang/py3/avro/tests/av_bench.py
+++ b/lang/py3/avro/tests/av_bench.py
@@ -28,7 +28,6 @@ import avro.datafile
import avro.io
import avro.schema
-
TYPES = ('A', 'CNAME',)
FILENAME = 'datafile.avr'
diff --git a/lang/py3/avro/tests/gen_interop_data.py b/lang/py3/avro/tests/gen_interop_data.py
index 4b83c30..c76f2c2 100644
--- a/lang/py3/avro/tests/gen_interop_data.py
+++ b/lang/py3/avro/tests/gen_interop_data.py
@@ -20,10 +20,7 @@
import sys
-from avro import datafile
-from avro import io
-from avro import schema
-
+from avro import datafile, io, schema
DATUM = {
'intField': 12,
diff --git a/lang/py3/avro/tests/run_tests.py b/lang/py3/avro/tests/run_tests.py
index 2111ec6..2575b36 100644
--- a/lang/py3/avro/tests/run_tests.py
+++ b/lang/py3/avro/tests/run_tests.py
@@ -49,12 +49,12 @@ import unittest
from avro.tests.test_datafile import *
from avro.tests.test_datafile_interop import *
+from avro.tests.test_enum import *
from avro.tests.test_io import *
from avro.tests.test_ipc import *
from avro.tests.test_protocol import *
from avro.tests.test_schema import *
from avro.tests.test_script import *
-from avro.tests.test_enum import *
def SetupLogging():
diff --git a/lang/py3/avro/tests/sample_http_client.py b/lang/py3/avro/tests/sample_http_client.py
index 574d8c7..da8d030 100644
--- a/lang/py3/avro/tests/sample_http_client.py
+++ b/lang/py3/avro/tests/sample_http_client.py
@@ -19,8 +19,7 @@
# limitations under the License.
import sys
-from avro import ipc
-from avro import protocol
+from avro import ipc, protocol
MAIL_PROTOCOL_JSON = """\
{"namespace": "example.proto",
diff --git a/lang/py3/avro/tests/sample_http_server.py b/lang/py3/avro/tests/sample_http_server.py
index fd389d2..72518be 100644
--- a/lang/py3/avro/tests/sample_http_server.py
+++ b/lang/py3/avro/tests/sample_http_server.py
@@ -17,9 +17,10 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
-from avro import ipc
-from avro import protocol
+
+from avro import ipc, protocol
MAIL_PROTOCOL_JSON = """\
{"namespace": "example.proto",
diff --git a/lang/py3/avro/tests/test_datafile.py b/lang/py3/avro/tests/test_datafile.py
index d5b07df..aa4042f 100644
--- a/lang/py3/avro/tests/test_datafile.py
+++ b/lang/py3/avro/tests/test_datafile.py
@@ -23,10 +23,18 @@ import os
import tempfile
import unittest
-from avro import datafile
-from avro import io
-from avro import schema
+from avro import datafile, io, schema
+try:
+ import snappy
+ HAS_SNAPPY = True
+except ImportError:
+ HAS_SNAPPY = False
+try:
+ import zstandard
+ HAS_ZSTANDARD = True
+except ImportError:
+ HAS_ZSTANDARD = False
# ------------------------------------------------------------------------------
diff --git a/lang/py3/avro/tests/test_enum.py b/lang/py3/avro/tests/test_enum.py
index 2db1f9e..040b3cb 100644
--- a/lang/py3/avro/tests/test_enum.py
+++ b/lang/py3/avro/tests/test_enum.py
@@ -22,6 +22,7 @@ import unittest
from avro import schema
+
class TestEnum(unittest.TestCase):
def testSymbolsInOrder(self):
enum = schema.EnumSchema('Test', '', ['A', 'B'], schema.Names(), '', {})
diff --git a/lang/py3/avro/tests/test_io.py b/lang/py3/avro/tests/test_io.py
index 6e44ee2..2fcd689 100644
--- a/lang/py3/avro/tests/test_io.py
+++ b/lang/py3/avro/tests/test_io.py
@@ -27,7 +27,6 @@ import unittest
from avro import io as avro_io
from avro import schema
-
SCHEMAS_TO_VALIDATE = (
('"null"', None),
('"boolean"', True),
diff --git a/lang/py3/avro/tests/test_ipc.py b/lang/py3/avro/tests/test_ipc.py
index 225f4d0..20eee1a 100644
--- a/lang/py3/avro/tests/test_ipc.py
+++ b/lang/py3/avro/tests/test_ipc.py
@@ -27,9 +27,7 @@ import threading
import time
import unittest
-from avro import ipc
-from avro import protocol
-from avro import schema
+from avro import ipc, protocol, schema
def NowMS():
diff --git a/lang/py3/avro/tests/test_normalization.py b/lang/py3/avro/tests/test_normalization.py
index 8935529..148bfc0 100644
--- a/lang/py3/avro/tests/test_normalization.py
+++ b/lang/py3/avro/tests/test_normalization.py
@@ -21,7 +21,7 @@
import unittest
from avro.schema import Parse
-from avro.schemanormalization import ToParsingCanonicalForm, Fingerprint, FingerprintAlgorithmNames
+from avro.schemanormalization import Fingerprint, FingerprintAlgorithmNames, ToParsingCanonicalForm
class TestSchemaNormalization(unittest.TestCase):
diff --git a/lang/py3/avro/tests/test_protocol.py b/lang/py3/avro/tests/test_protocol.py
index 001db6b..abd17e8 100644
--- a/lang/py3/avro/tests/test_protocol.py
+++ b/lang/py3/avro/tests/test_protocol.py
@@ -27,7 +27,6 @@ import unittest
from avro import protocol
-
# ------------------------------------------------------------------------------
diff --git a/lang/py3/avro/tests/test_schema.py b/lang/py3/avro/tests/test_schema.py
index c2759da..2d82904 100644
--- a/lang/py3/avro/tests/test_schema.py
+++ b/lang/py3/avro/tests/test_schema.py
@@ -27,7 +27,6 @@ import unittest
from avro import schema
-
# ------------------------------------------------------------------------------
diff --git a/lang/py3/avro/tests/test_script.py b/lang/py3/avro/tests/test_script.py
index 1b267a1..647d346 100644
--- a/lang/py3/avro/tests/test_script.py
+++ b/lang/py3/avro/tests/test_script.py
@@ -32,7 +32,6 @@ import avro.datafile
import avro.io
import avro.schema
-
# ------------------------------------------------------------------------------
diff --git a/lang/py3/avro/tests/txsample_http_client.py b/lang/py3/avro/tests/txsample_http_client.py
index 39c583a..fede960 100644
--- a/lang/py3/avro/tests/txsample_http_client.py
+++ b/lang/py3/avro/tests/txsample_http_client.py
@@ -19,12 +19,10 @@
# limitations under the License.
import sys
-from twisted.internet import reactor, defer
+from avro import protocol, txipc
+from twisted.internet import defer, reactor
from twisted.python.util import println
-from avro import protocol
-from avro import txipc
-
MAIL_PROTOCOL_JSON = """\
{"namespace": "example.proto",
"protocol": "Mail",
diff --git a/lang/py3/avro/tests/txsample_http_server.py b/lang/py3/avro/tests/txsample_http_server.py
index 1936cc6..c85f42a 100644
--- a/lang/py3/avro/tests/txsample_http_server.py
+++ b/lang/py3/avro/tests/txsample_http_server.py
@@ -13,16 +13,15 @@
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
+
+
+from avro import ipc, protocol, txipc
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+from twisted.internet import reactor
# See the License for the specific language governing permissions and
# limitations under the License.
from twisted.web import server
-from twisted.internet import reactor
-
-from avro import ipc
-from avro import protocol
-from avro import txipc
MAIL_PROTOCOL_JSON = """\
{"namespace": "example.proto",
diff --git a/lang/py3/avro/tool.py b/lang/py3/avro/tool.py
index 28bf0dc..6bfce0d 100644
--- a/lang/py3/avro/tool.py
+++ b/lang/py3/avro/tool.py
@@ -25,13 +25,10 @@ NOTE: The API for the command-line tool is experimental.
import sys
import urllib
+from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
-from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
+from avro import datafile, io, ipc, protocol
-from avro import io
-from avro import datafile
-from avro import protocol
-from avro import ipc
class GenericResponder(ipc.Responder):
def __init__(self, proto, msg, datum):
diff --git a/lang/py3/avro/txipc.py b/lang/py3/avro/txipc.py
index 47a6bfe..d8e3225 100644
--- a/lang/py3/avro/txipc.py
+++ b/lang/py3/avro/txipc.py
@@ -20,12 +20,11 @@
import io
-from avro import io as avro_io
-from avro import ipc
-
from zope.interface import implements
-from twisted.internet.defer import maybeDeferred, Deferred
+from avro import io as avro_io
+from avro import ipc
+from twisted.internet.defer import Deferred, maybeDeferred
from twisted.internet.protocol import Protocol
from twisted.web import resource, server
from twisted.web.client import Agent
diff --git a/lang/py3/build.sh b/lang/py3/build.sh
index 1bb27ad..fc4564f 100755
--- a/lang/py3/build.sh
+++ b/lang/py3/build.sh
@@ -18,7 +18,7 @@
set -e # exit on error
usage() {
- echo "Usage: $0 {test|dist|clean}"
+ echo "Usage: $0 {isort|lint|test|dist|clean}"
}
main() {
@@ -29,8 +29,9 @@ main() {
fi
for target; do
case "$target" in
- clean|dist|test) : ;;
- *) usage; return 1 ;;
+ lint) set -- isort "$@";;
+ clean|dist|isort|test) :;;
+ *) usage; return 1;;
esac
done
python3 setup.py "$@"
diff --git a/lang/py3/setup.cfg b/lang/py3/setup.cfg
index affeded..108c740 100644
--- a/lang/py3/setup.cfg
+++ b/lang/py3/setup.cfg
@@ -62,3 +62,12 @@ snappy = snappy
[aliases]
dist = sdist --dist-dir ../../dist/py3
+
+[isort]
+line_length = 150
+known_third_party=zope
+
+[pycodestyle]
+max-line-length = 150
+statistics = True
+ignore = E111,E114,E121,E122,E124,E127,E128,E129,E201,E202,E203,E221,E225,E231,E241,E261,E301,E302,E303,E305,E402,E701,E703,W503
diff --git a/lang/py3/setup.py b/lang/py3/setup.py
index a752779..130ce18 100755
--- a/lang/py3/setup.py
+++ b/lang/py3/setup.py
@@ -27,13 +27,17 @@ https://pypi.org/project/avro-python3/
"""
import distutils.command.clean
-import distutils.file_util
import distutils.dir_util
+import distutils.errors
+import distutils.file_util
import distutils.log
import fnmatch
import os
+import subprocess
-from setuptools import setup
+import setuptools
+
+import pycodestyle
_HERE = os.path.dirname(os.path.abspath(__file__))
_AVRO_DIR = os.path.join(_HERE, 'avro')
@@ -110,11 +114,53 @@ class CleanCommand(distutils.command.clean.clean):
os.remove(name)
+class GenerateInteropDataCommand(setuptools.Command):
+ """A command to generate Avro files for data interop test."""
+
+ user_options = [
+ ('schema-file=', None, 'path to input Avro schema file'),
+ ('output-path=', None, 'path to output Avro data files'),
+ ]
+
+ def initialize_options(self):
+ self.schema_file = os.path.join(os.getcwd(), 'interop.avsc')
+ self.output_path = os.getcwd()
+
+ def finalize_options(self):
+ pass
+
+ def run(self):
+ from avro.tests import gen_interop_data
+ gen_interop_data.generate(self.schema_file, self.output_path)
+
+
+class LintCommand(setuptools.Command):
+ """Run pycodestyle on all your modules"""
+ description = __doc__
+ user_options = []
+
+ def initialize_options(self):
+ pass
+
+ def finalize_options(self):
+ pass
+
+ def run(self):
+ try:
+ subprocess.run(['pycodestyle', '.'], check=True)
+ except subprocess.CalledProcessError:
+ raise distutils.errors.DistutilsError("pycodestyle exited with a nonzero exit code.")
+
+
def main():
if not _is_distribution():
_generate_package_data()
- setup(cmdclass={"clean": CleanCommand})
+ setuptools.setup(cmdclass={
+ "clean": CleanCommand,
+ "generate_interop_data": GenerateInteropDataCommand,
+ "lint": LintCommand,
+ })
if __name__ == '__main__':
diff --git a/lang/ruby/build.sh b/lang/ruby/build.sh
index ebcd682..1f1319d 100755
--- a/lang/ruby/build.sh
+++ b/lang/ruby/build.sh
@@ -29,23 +29,24 @@ gem install --no-document -v 1.17.3 bundler
bundle install
case "$1" in
- test)
- bundle exec rake test
- ;;
-
- dist)
- bundle exec rake dist
- ;;
-
- clean)
- bundle exec rake clean
- rm -rf tmp avro.gemspec data.avr
- ;;
-
- *)
- echo "Usage: $0 {test|dist|clean}"
- exit 1
-
+ lint)
+ echo 'This is a stub where someone can provide linting.'
+ ;;
+
+ test)
+ bundle exec rake test
+ ;;
+
+ dist)
+ bundle exec rake dist
+ ;;
+
+ clean)
+ bundle exec rake clean
+ rm -rf tmp avro.gemspec data.avr
+ ;;
+
+ *)
+ echo "Usage: $0 {lint|test|dist|clean}"
+ exit 1
esac
-
-exit 0
diff --git a/share/docker/Dockerfile b/share/docker/Dockerfile
index 72a6811..c6944ce 100644
--- a/share/docker/Dockerfile
+++ b/share/docker/Dockerfile
@@ -53,6 +53,7 @@ RUN apt-get -qq update && \
g++ \
gcc \
git \
+ isort \
libboost-all-dev \
libfontconfig1-dev \
libfreetype6-dev \
@@ -66,8 +67,15 @@ RUN apt-get -qq update && \
perl \
php5.6 \
php5.6-gmp \
+ pycodestyle \
python \
+ python-isort \
+ python-pip \
python-setuptools \
+ python-snappy \
+ python-wheel \
+ python3-isort \
+ python3-pip \
python3-setuptools \
rake \
ruby \
diff --git a/share/docker/run-tests.sh b/share/docker/run-tests.sh
index bf13887..a585a41 100755
--- a/share/docker/run-tests.sh
+++ b/share/docker/run-tests.sh
@@ -27,5 +27,5 @@ for lang in /avro/lang/*/
do
headline "Run tests: $lang"
cd "$lang"
- ./build.sh test
+ ./build.sh lint test
done
[avro] 16/23: Undo some language tool bumps.
Posted by rs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rskraba pushed a commit to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/avro.git
commit d9ef6379654f3e2ff00424ae5c1c9db70b1b755d
Author: Ryan Skraba <ry...@skraba.com>
AuthorDate: Wed Jan 22 16:01:05 2020 +0100
Undo some language tool bumps.
The cherry-pick for AVRO-2590 reorganizes the language tools in docker,
as well as fixing the .NET installation. This commit keeps the
refactoring but undoes some bumps for PHP, Ruby and python setup.
---
share/docker/Dockerfile | 13 ++++---------
1 file changed, 4 insertions(+), 9 deletions(-)
diff --git a/share/docker/Dockerfile b/share/docker/Dockerfile
index 926b270..8d73b67 100644
--- a/share/docker/Dockerfile
+++ b/share/docker/Dockerfile
@@ -30,7 +30,6 @@ RUN apt-get -qqy update \
asciidoc \
bison \
bzip2 \
- bundler \
cmake \
curl \
doxygen \
@@ -48,6 +47,7 @@ RUN apt-get -qqy update \
make \
maven \
perl \
+ pycodestyle \
python \
python-pip \
python-setuptools \
@@ -94,15 +94,11 @@ RUN curl -sSL https://packages.sury.org/php/apt.gpg \
&& apt-get -qqy update \
&& apt-get -qqy install --no-install-recommends php${PHP5_VERSION} \
php${PHP5_VERSION}-gmp \
- php${PHP7_VERSION} \
- php${PHP7_VERSION}-gmp \
&& apt-get -qqy clean \
&& rm -rf /var/lib/apt/lists
-# Install PHPUnit 5.7, which is the only version that supports both PHP 5.6 and 7.x
+# TODO: Install PHPUnit 5.7, which is the only version that supports both PHP 5.6 and 7.x
# TODO: use PHPUnit 7.x instead of 5.7 when we drop PHP 5.6 support in the future
-RUN wget -O /usr/local/bin/phpunit https://phar.phpunit.de/phpunit-5.7.phar && chmod +x /usr/local/bin/phpunit
-
-RUN curl -sSL https://phar.phpunit.de/phpunit-5.7.phar > /usr/local/bin/phpunit \
+RUN curl -sSL https://phar.phpunit.de/phpunit-5.6.phar > /usr/local/bin/phpunit \
&& chmod +x /usr/local/bin/phpunit
# Install Perl modules
@@ -138,8 +134,7 @@ RUN python3 -m pip install --upgrade pip setuptools \
&& python3 -m pip install zstandard
# Install Ruby modules
-COPY lang/ruby/Gemfile /tmp
-RUN bundle install --gemfile=/tmp/Gemfile
+RUN gem install echoe yajl-ruby multi_json snappy
# Install .NET SDK
RUN curl -sSLO https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb \
[avro] 18/23: AVRO-2601: zstandard in extras_require
Posted by rs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rskraba pushed a commit to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/avro.git
commit 075a12a10ae30aba0187b225ca7669b97c1ce1ea
Author: Michael A. Smith <mi...@smith-li.com>
AuthorDate: Sun Oct 20 11:53:04 2019 -0400
AVRO-2601: zstandard in extras_require
---
lang/py/setup.py | 1 +
lang/py3/setup.cfg | 1 +
2 files changed, 2 insertions(+)
diff --git a/lang/py/setup.py b/lang/py/setup.py
index 6063869..c2e07c6 100755
--- a/lang/py/setup.py
+++ b/lang/py/setup.py
@@ -84,5 +84,6 @@ setuptools.setup(
url = 'https://avro.apache.org/',
extras_require = {
'snappy': ['python-snappy'],
+ 'zstandard': ['zstandard'],
},
)
diff --git a/lang/py3/setup.cfg b/lang/py3/setup.cfg
index 28fcd45..c721ca6 100644
--- a/lang/py3/setup.cfg
+++ b/lang/py3/setup.cfg
@@ -62,6 +62,7 @@ avro =
[options.extras_require]
snappy = snappy
+zstandard = zstandard
[aliases]
dist = sdist --dist-dir ../../dist/py3
[avro] 03/23: AVRO-2413: Refactor io.Validate to use a mapping
(#516)
Posted by rs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rskraba pushed a commit to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/avro.git
commit 3fc05a3800318a405aab00e345886e9885dd1339
Author: Michael A. Smith <mi...@syapse.com>
AuthorDate: Mon Jun 24 05:27:14 2019 -0400
AVRO-2413: Refactor io.Validate to use a mapping (#516)
* AVRO-2413: Refactor Validate to Use a Mapping
* AVRO-2443: Fix Invalid AvroTypeException Invocation
---
lang/py/src/avro/io.py | 112 +++++++++++++++++++++++--------------------------
lang/py3/avro/io.py | 71 +++++++++++--------------------
2 files changed, 78 insertions(+), 105 deletions(-)
diff --git a/lang/py/src/avro/io.py b/lang/py/src/avro/io.py
index 3fdf602..8c2ef10 100644
--- a/lang/py/src/avro/io.py
+++ b/lang/py/src/avro/io.py
@@ -5,9 +5,9 @@
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
-#
+#
# https://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -34,7 +34,7 @@ uses the following mapping:
* Schema longs are implemented as long.
* Schema floats are implemented as float.
* Schema doubles are implemented as float.
- * Schema booleans are implemented as bool.
+ * Schema booleans are implemented as bool.
"""
import struct
from avro import schema
@@ -106,65 +106,59 @@ class SchemaResolutionException(schema.AvroException):
#
# Validate
#
-
def _is_timezone_aware_datetime(dt):
return dt.tzinfo is not None and dt.tzinfo.utcoffset(dt) is not None
+_valid = {
+ 'null': lambda s, d: d is None,
+ 'boolean': lambda s, d: isinstance(d, bool),
+ 'string': lambda s, d: isinstance(d, basestring),
+ 'bytes': lambda s, d: ((isinstance(d, str)) or
+ (isinstance(d, Decimal) and
+ getattr(s, 'logical_type', None) == constants.DECIMAL)),
+ 'int': lambda s, d: ((isinstance(d, (int, long))) and (INT_MIN_VALUE <= d <= INT_MAX_VALUE) or
+ (isinstance(d, datetime.date) and
+ getattr(s, 'logical_type', None) == constants.DATE) or
+ (isinstance(d, datetime.time) and
+ getattr(s, 'logical_type', None) == constants.TIME_MILLIS)),
+ 'long': lambda s, d: ((isinstance(d, (int, long))) and (LONG_MIN_VALUE <= d <= LONG_MAX_VALUE) or
+ (isinstance(d, datetime.time) and
+ getattr(s, 'logical_type', None) == constants.TIME_MICROS) or
+ (isinstance(d, datetime.date) and
+ _is_timezone_aware_datetime(d) and
+ getattr(s, 'logical_type', None) in (constants.TIMESTAMP_MILLIS,
+ constants.TIMESTAMP_MICROS))),
+ 'float': lambda s, d: isinstance(d, (int, long, float)),
+ 'fixed': lambda s, d: ((isinstance(d, str) and len(d) == s.size) or
+ (isinstance(d, Decimal) and
+ getattr(s, 'logical_type', None) == constants.DECIMAL)),
+ 'enum': lambda s, d: d in s.symbols,
+ 'array': lambda s, d: isinstance(d, list) and all(validate(s.items, item) for item in d),
+ 'map': lambda s, d: (isinstance(d, dict) and all(isinstance(key, basestring) for key in d)
+ and all(validate(s.values, value) for value in d.values())),
+ 'union': lambda s, d: any(validate(branch, d) for branch in s.schemas),
+ 'record': lambda s, d: (isinstance(d, dict)
+ and all(validate(f.type, d.get(f.name)) for f in s.fields)
+ and {f.name for f in s.fields}.issuperset(d.keys())),
+}
+_valid['double'] = _valid['float']
+_valid['error_union'] = _valid['union']
+_valid['error'] = _valid['request'] = _valid['record']
+
+
def validate(expected_schema, datum):
- """Determine if a python datum is an instance of a schema."""
- schema_type = expected_schema.type
- if schema_type == 'null':
- return datum is None
- elif schema_type == 'boolean':
- return isinstance(datum, bool)
- elif schema_type == 'string':
- return isinstance(datum, basestring)
- elif schema_type == 'bytes':
- if (hasattr(expected_schema, 'logical_type') and
- expected_schema.logical_type == 'decimal'):
- return isinstance(datum, Decimal)
- return isinstance(datum, str)
- elif schema_type == 'int':
- if hasattr(expected_schema, 'logical_type'):
- if expected_schema.logical_type == constants.DATE:
- return isinstance(datum, datetime.date)
- elif expected_schema.logical_type == constants.TIME_MILLIS:
- return isinstance(datum, datetime.time)
- return (isinstance(datum, (int, long))
- and INT_MIN_VALUE <= datum <= INT_MAX_VALUE)
- elif schema_type == 'long':
- if hasattr(expected_schema, 'logical_type'):
- if expected_schema.logical_type == constants.TIME_MICROS:
- return isinstance(datum, datetime.time)
- elif expected_schema.logical_type in [constants.TIMESTAMP_MILLIS, constants.TIMESTAMP_MICROS]:
- return isinstance(datum, datetime.datetime) and _is_timezone_aware_datetime(datum)
- return (isinstance(datum, (int, long))
- and LONG_MIN_VALUE <= datum <= LONG_MAX_VALUE)
- elif schema_type in ['float', 'double']:
- return (isinstance(datum, int) or isinstance(datum, long)
- or isinstance(datum, float))
- # Check for int, float, long and decimal
- elif schema_type == 'fixed':
- if (hasattr(expected_schema, 'logical_type') and
- expected_schema.logical_type == 'decimal'):
- return isinstance(datum, Decimal)
- return isinstance(datum, str) and len(datum) == expected_schema.size
- elif schema_type == 'enum':
- return datum in expected_schema.symbols
- elif schema_type == 'array':
- return (isinstance(datum, list) and
- False not in [validate(expected_schema.items, d) for d in datum])
- elif schema_type == 'map':
- return (isinstance(datum, dict) and
- False not in [isinstance(k, basestring) for k in datum.keys()] and
- False not in
- [validate(expected_schema.values, v) for v in datum.values()])
- elif schema_type in ['union', 'error_union']:
- return True in [validate(s, datum) for s in expected_schema.schemas]
- elif schema_type in ['record', 'error', 'request']:
- return (isinstance(datum, dict) and
- False not in
- [validate(f.type, datum.get(f.name)) for f in expected_schema.fields])
+ """Determines if a python datum is an instance of a schema.
+
+ Args:
+ expected_schema: Schema to validate against.
+ datum: Datum to validate.
+ Returns:
+ True if the datum is an instance of the schema.
+ """
+ try:
+ return _valid[expected_schema.type](expected_schema, datum)
+ except KeyError:
+ raise AvroTypeException('Unknown Avro schema type: %r' % schema_type)
#
# Decoder/Encoder
@@ -1040,7 +1034,7 @@ class DatumWriter(object):
# validate datum
if not validate(self.writers_schema, datum):
raise AvroTypeException(self.writers_schema, datum)
-
+
self.write_data(self.writers_schema, datum, encoder)
def write_data(self, writers_schema, datum, encoder):
diff --git a/lang/py3/avro/io.py b/lang/py3/avro/io.py
index 9fe7f09..cd79393 100644
--- a/lang/py3/avro/io.py
+++ b/lang/py3/avro/io.py
@@ -92,6 +92,27 @@ class SchemaResolutionException(schema.AvroException):
# ------------------------------------------------------------------------------
# Validate
+_valid = {
+ 'null': lambda s, d: d is None,
+ 'boolean': lambda s, d: isinstance(d, bool),
+ 'string': lambda s, d: isinstance(d, str),
+ 'bytes': lambda s, d: isinstance(d, bytes),
+ 'int': lambda s, d: isinstance(d, int) and (INT_MIN_VALUE <= d <= INT_MAX_VALUE),
+ 'long': lambda s, d: isinstance(d, int) and (LONG_MIN_VALUE <= d <= LONG_MAX_VALUE),
+ 'float': lambda s, d: isinstance(d, (int, float)),
+ 'fixed': lambda s, d: isinstance(d, bytes) and len(d) == s.size,
+ 'enum': lambda s, d: d in s.symbols,
+ 'array': lambda s, d: isinstance(d, list) and all(Validate(s.items, item) for item in d),
+ 'map': lambda s, d: (isinstance(d, dict) and all(isinstance(key, str) for key in d)
+ and all(Validate(s.values, value) for value in d.values())),
+ 'union': lambda s, d: any(Validate(branch, d) for branch in s.schemas),
+ 'record': lambda s, d: (isinstance(d, dict)
+ and all(Validate(f.type, d.get(f.name)) for f in s.fields)
+ and {f.name for f in s.fields}.issuperset(d.keys()))
+}
+_valid['double'] = _valid['float']
+_valid['error_union'] = _valid['union']
+_valid['error'] = _valid['request'] = _valid['record']
def Validate(expected_schema, datum):
"""Determines if a python datum is an instance of a schema.
@@ -102,52 +123,10 @@ def Validate(expected_schema, datum):
Returns:
True if the datum is an instance of the schema.
"""
- schema_type = expected_schema.type
- if schema_type == 'null':
- return datum is None
- elif schema_type == 'boolean':
- return isinstance(datum, bool)
- elif schema_type == 'string':
- return isinstance(datum, str)
- elif schema_type == 'bytes':
- return isinstance(datum, bytes)
- elif schema_type == 'int':
- return (isinstance(datum, int)
- and (INT_MIN_VALUE <= datum <= INT_MAX_VALUE))
- elif schema_type == 'long':
- return (isinstance(datum, int)
- and (LONG_MIN_VALUE <= datum <= LONG_MAX_VALUE))
- elif schema_type in ['float', 'double']:
- return (isinstance(datum, int) or isinstance(datum, float))
- elif schema_type == 'fixed':
- return isinstance(datum, bytes) and (len(datum) == expected_schema.size)
- elif schema_type == 'enum':
- return datum in expected_schema.symbols
- elif schema_type == 'array':
- return (isinstance(datum, list)
- and all(Validate(expected_schema.items, item) for item in datum))
- elif schema_type == 'map':
- return (isinstance(datum, dict)
- and all(isinstance(key, str) for key in datum.keys())
- and all(Validate(expected_schema.values, value)
- for value in datum.values()))
- elif schema_type in ['union', 'error_union']:
- return any(Validate(union_branch, datum)
- for union_branch in expected_schema.schemas)
- elif schema_type in ['record', 'error', 'request']:
- if not isinstance(datum, dict):
- return False
- expected_schema_field_names = set()
- for field in expected_schema.fields:
- expected_schema_field_names.add(field.name)
- if not Validate(field.type, datum.get(field.name)):
- return False
- for datum_field in datum.keys():
- if datum_field not in expected_schema_field_names:
- return False
- return True
- else:
- raise AvroTypeException('Unknown Avro schema type: %r' % schema_type)
+ try:
+ return _valid[expected_schema.type](expected_schema, datum)
+ except KeyError:
+ raise AvroTypeException(expected_schema, datum)
# ------------------------------------------------------------------------------
[avro] 02/23: AVRO-2388: Drop Python 3.4 Support (#513)
Posted by rs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rskraba pushed a commit to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/avro.git
commit a4945154bb99503a94b0f387cf1f28931781a580
Author: Michael A. Smith <mi...@syapse.com>
AuthorDate: Mon Jun 3 09:12:21 2019 -0400
AVRO-2388: Drop Python 3.4 Support (#513)
---
lang/py3/setup.cfg | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/lang/py3/setup.cfg b/lang/py3/setup.cfg
index 9d1f437..affeded 100644
--- a/lang/py3/setup.cfg
+++ b/lang/py3/setup.cfg
@@ -32,7 +32,6 @@ license = Apache License 2.0
classifiers =
License :: OSI Approved :: Apache Software License
Programming Language :: Python :: 3 :: Only
- Programming Language :: Python :: 3.4
Programming Language :: Python :: 3.5
Programming Language :: Python :: 3.6
Programming Language :: Python :: 3.7
@@ -47,7 +46,7 @@ install_requires =
zip_safe = true
scripts =
scripts/avro
-python_requires = >=3.4
+python_requires = >=3.5
test_suite = avro.tests.run_tests
[options.package_data]
[avro] 19/23: AVRO-1788: Remove Obsolete Python < 2.7 Syntax (#683)
Posted by rs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rskraba pushed a commit to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/avro.git
commit e9eee386a14b592a7cca55a1214c14c10d03a3b2
Author: Michael A. Smith <mi...@smith-li.com>
AuthorDate: Tue Oct 22 08:49:44 2019 -0400
AVRO-1788: Remove Obsolete Python < 2.7 Syntax (#683)
---
lang/py/scripts/avro | 24 +++++----
lang/py/setup.py | 7 ++-
lang/py/src/avro/__init__.py | 10 ++--
lang/py/src/avro/constants.py | 13 +++--
lang/py/src/avro/datafile.py | 2 +-
lang/py/src/avro/io.py | 63 ++++++++++++----------
lang/py/src/avro/ipc.py | 27 ++++++----
lang/py/src/avro/protocol.py | 32 +++++------
lang/py/src/avro/schema.py | 31 +++++++----
lang/py/src/avro/tether/__init__.py | 15 +++---
lang/py/src/avro/tether/tether_task.py | 41 +++++++-------
lang/py/src/avro/tether/tether_task_runner.py | 61 ++++++++++-----------
lang/py/src/avro/tether/util.py | 37 ++++++-------
lang/py/src/avro/timezones.py | 9 +++-
lang/py/src/avro/tool.py | 45 +++++++++-------
lang/py/src/avro/txipc.py | 13 +++--
lang/py/{src/avro => test}/__init__.py | 10 ++--
lang/py/test/av_bench.py | 7 ++-
lang/py/test/gen_interop_data.py | 40 +++++++++++---
lang/py/test/mock_tether_parent.py | 36 ++++++-------
lang/py/test/sample_http_client.py | 7 ++-
lang/py/test/sample_http_server.py | 3 ++
lang/py/test/set_avro_test_path.py | 7 +++
lang/py/test/test_datafile.py | 77 +++++++++++++++------------
lang/py/test/test_datafile_interop.py | 43 +++++++++------
lang/py/test/test_io.py | 59 ++++++++++----------
lang/py/test/test_ipc.py | 7 +++
lang/py/test/test_script.py | 11 ++--
lang/py/test/test_tether_task.py | 39 ++++++++------
lang/py/test/test_tether_task_runner.py | 67 ++++++++++++-----------
lang/py/test/test_tether_word_count.py | 15 +++---
lang/py/test/txsample_http_client.py | 5 ++
lang/py/test/txsample_http_server.py | 3 ++
lang/py/test/word_count_task.py | 49 +++++++++--------
34 files changed, 533 insertions(+), 382 deletions(-)
diff --git a/lang/py/scripts/avro b/lang/py/scripts/avro
index b320a39..1aac028 100644
--- a/lang/py/scripts/avro
+++ b/lang/py/scripts/avro
@@ -18,16 +18,19 @@
"""Command line utility for reading and writing Avro files."""
-from avro.io import DatumReader, DatumWriter
-from avro.datafile import DataFileReader, DataFileWriter
-import avro.schema
+from __future__ import absolute_import, division, print_function
-import json
import csv
-from sys import stdout, stdin
-from itertools import ifilter, imap
+import json
from functools import partial
+from itertools import ifilter, imap
from os.path import splitext
+from sys import stdin, stdout
+
+import avro.schema
+from avro.datafile import DataFileReader, DataFileWriter
+from avro.io import DatumReader, DatumWriter
+
class AvroError(Exception):
pass
@@ -52,9 +55,9 @@ def print_csv(row):
def select_printer(format):
return {
- "json" : print_json,
- "json-pretty" : print_json_pretty,
- "csv" : print_csv
+ "json": print_json,
+ "json-pretty": print_json_pretty,
+ "csv": print_csv
}[format]
def record_match(expr, record):
@@ -102,7 +105,7 @@ def print_avro(avro, opts):
def print_schema(avro):
schema = avro.meta["avro.schema"]
# Pretty print
- print json.dumps(json.loads(schema), indent=4)
+ print(json.dumps(json.loads(schema), indent=4))
def cat(opts, args):
if not args:
@@ -257,4 +260,3 @@ def main(argv=None):
if __name__ == "__main__":
main()
-
diff --git a/lang/py/setup.py b/lang/py/setup.py
index c2e07c6..b978092 100755
--- a/lang/py/setup.py
+++ b/lang/py/setup.py
@@ -1,5 +1,6 @@
#!/usr/bin/env python
+##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -7,9 +8,9 @@
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
-#
+#
# https://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -17,6 +18,8 @@
# limitations under the License.
+from __future__ import absolute_import, division, print_function
+
import distutils.errors
import glob
import os
diff --git a/lang/py/src/avro/__init__.py b/lang/py/src/avro/__init__.py
index 47d1295..9a859e9 100644
--- a/lang/py/src/avro/__init__.py
+++ b/lang/py/src/avro/__init__.py
@@ -1,3 +1,6 @@
+#!/usr/bin/env python
+
+##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -5,14 +8,15 @@
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
-#
+#
# https://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-__all__ = ['schema', 'io', 'datafile', 'protocol', 'ipc', 'constants', 'timezones']
+from __future__ import absolute_import, division, print_function
+__all__ = ['schema', 'io', 'datafile', 'protocol', 'ipc', 'constants', 'timezones']
diff --git a/lang/py/src/avro/constants.py b/lang/py/src/avro/constants.py
index 66e31df..2197201 100644
--- a/lang/py/src/avro/constants.py
+++ b/lang/py/src/avro/constants.py
@@ -1,3 +1,6 @@
+#!/usr/bin/env python
+
+##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -5,18 +8,18 @@
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
-#
+#
# https://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-"""
-Contains Constants for Python Avro
-"""
+"""Contains Constants for Python Avro"""
+
+from __future__ import absolute_import, division, print_function
DATE = "date"
DECIMAL = "decimal"
diff --git a/lang/py/src/avro/datafile.py b/lang/py/src/avro/datafile.py
index 75a8e4a..a9c9c22 100644
--- a/lang/py/src/avro/datafile.py
+++ b/lang/py/src/avro/datafile.py
@@ -401,4 +401,4 @@ def generate_sixteen_random_bytes():
try:
return os.urandom(16)
except NotImplementedError:
- return [chr(random.randrange(256)) for i in range(16)]
+ return bytes(random.randrange(256) for i in range(16))
diff --git a/lang/py/src/avro/io.py b/lang/py/src/avro/io.py
index c36c5b3..b18b148 100644
--- a/lang/py/src/avro/io.py
+++ b/lang/py/src/avro/io.py
@@ -1,3 +1,6 @@
+#!/usr/bin/env python
+
+##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -38,6 +41,8 @@ uses the following mapping:
* Schema booleans are implemented as bool.
"""
+from __future__ import absolute_import, division, print_function
+
import datetime
import json
import struct
@@ -183,7 +188,7 @@ class BinaryDecoder(object):
def read_boolean(self):
"""
- a boolean is written as a single byte
+ a boolean is written as a single byte
whose value is either 0 (false) or 1 (true).
"""
return ord(self.read(1)) == 1
@@ -261,7 +266,7 @@ class BinaryDecoder(object):
def read_bytes(self):
"""
- Bytes are encoded as a long followed by that many bytes of data.
+ Bytes are encoded as a long followed by that many bytes of data.
"""
return self.read(self.read_long())
@@ -297,7 +302,7 @@ class BinaryDecoder(object):
def read_time_millis_from_int(self):
"""
- int is decoded as python time object which represents
+ int is decoded as python time object which represents
the number of milliseconds after midnight, 00:00:00.000.
"""
milliseconds = self.read_int()
@@ -305,7 +310,7 @@ class BinaryDecoder(object):
def read_time_micros_from_long(self):
"""
- long is decoded as python time object which represents
+ long is decoded as python time object which represents
the number of microseconds after midnight, 00:00:00.000000.
"""
microseconds = self.read_long()
@@ -313,17 +318,17 @@ class BinaryDecoder(object):
def read_timestamp_millis_from_long(self):
"""
- long is decoded as python datetime object which represents
+ long is decoded as python datetime object which represents
the number of milliseconds from the unix epoch, 1 January 1970.
"""
timestamp_millis = self.read_long()
timedelta = datetime.timedelta(microseconds=timestamp_millis * 1000)
- unix_epoch_datetime = datetime.datetime(1970, 1, 1, 0, 0, 0, 0, tzinfo=timezones.utc)
+ unix_epoch_datetime = datetime.datetime(1970, 1, 1, 0, 0, 0, 0, tzinfo=timezones.utc)
return unix_epoch_datetime + timedelta
def read_timestamp_micros_from_long(self):
"""
- long is decoded as python datetime object which represents
+ long is decoded as python datetime object which represents
the number of microseconds from the unix epoch, 1 January 1970.
"""
timestamp_micros = self.read_long()
@@ -386,10 +391,10 @@ class BinaryEncoder(object):
null is written as zero bytes
"""
pass
-
+
def write_boolean(self, datum):
"""
- a boolean is written as a single byte
+ a boolean is written as a single byte
whose value is either 0 (false) or 1 (true).
"""
if datum:
@@ -399,7 +404,7 @@ class BinaryEncoder(object):
def write_int(self, datum):
"""
- int and long values are written using variable-length, zig-zag coding.
+ int and long values are written using variable-length, zig-zag coding.
"""
self.write_long(datum);
@@ -491,7 +496,7 @@ class BinaryEncoder(object):
bits_to_write = unscaled_datum >> (8 * index)
self.write(chr(bits_to_write & 0xff))
else:
- for i in range(offset_bits/8):
+ for i in range(offset_bits // 8):
self.write(chr(0))
for index in range(bytes_req-1, -1, -1):
bits_to_write = unscaled_datum >> (8 * index)
@@ -499,7 +504,7 @@ class BinaryEncoder(object):
def write_bytes(self, datum):
"""
- Bytes are encoded as a long followed by that many bytes of data.
+ Bytes are encoded as a long followed by that many bytes of data.
"""
self.write_long(len(datum))
self.write(struct.pack('%ds' % len(datum), datum))
@@ -591,32 +596,32 @@ class DatumReader(object):
and w_type == r_type):
return True
elif (w_type == r_type == 'record' and
- DatumReader.check_props(writers_schema, readers_schema,
+ DatumReader.check_props(writers_schema, readers_schema,
['fullname'])):
return True
elif (w_type == r_type == 'error' and
- DatumReader.check_props(writers_schema, readers_schema,
+ DatumReader.check_props(writers_schema, readers_schema,
['fullname'])):
return True
elif (w_type == r_type == 'request'):
return True
- elif (w_type == r_type == 'fixed' and
- DatumReader.check_props(writers_schema, readers_schema,
+ elif (w_type == r_type == 'fixed' and
+ DatumReader.check_props(writers_schema, readers_schema,
['fullname', 'size'])):
return True
- elif (w_type == r_type == 'enum' and
- DatumReader.check_props(writers_schema, readers_schema,
+ elif (w_type == r_type == 'enum' and
+ DatumReader.check_props(writers_schema, readers_schema,
['fullname'])):
return True
- elif (w_type == r_type == 'map' and
+ elif (w_type == r_type == 'map' and
DatumReader.check_props(writers_schema.values,
readers_schema.values, ['type'])):
return True
- elif (w_type == r_type == 'array' and
+ elif (w_type == r_type == 'array' and
DatumReader.check_props(writers_schema.items,
readers_schema.items, ['type'])):
return True
-
+
# Handle schema promotion
if w_type == 'int' and r_type in ['long', 'float', 'double']:
return True
@@ -633,7 +638,7 @@ class DatumReader(object):
reader the "reader's schema".
"""
self._writers_schema = writers_schema
- self._readers_schema = readers_schema
+ self._readers_schema = readers_schema
# read/write properties
def set_writers_schema(self, writers_schema):
@@ -644,7 +649,7 @@ class DatumReader(object):
self._readers_schema = readers_schema
readers_schema = property(lambda self: self._readers_schema,
set_readers_schema)
-
+
def read(self, decoder):
if self.readers_schema is None:
self.readers_schema = self.writers_schema
@@ -682,13 +687,13 @@ class DatumReader(object):
else:
return decoder.read_int()
elif writers_schema.type == 'long':
- if (hasattr(writers_schema, 'logical_type') and
+ if (hasattr(writers_schema, 'logical_type') and
writers_schema.logical_type == constants.TIME_MICROS):
return decoder.read_time_micros_from_long()
- elif (hasattr(writers_schema, 'logical_type') and
+ elif (hasattr(writers_schema, 'logical_type') and
writers_schema.logical_type == constants.TIMESTAMP_MILLIS):
return decoder.read_timestamp_millis_from_long()
- elif (hasattr(writers_schema, 'logical_type') and
+ elif (hasattr(writers_schema, 'logical_type') and
writers_schema.logical_type == constants.TIMESTAMP_MICROS):
return decoder.read_timestamp_micros_from_long()
else:
@@ -886,7 +891,7 @@ class DatumReader(object):
% (index_of_schema, len(writers_schema.schemas))
raise SchemaResolutionException(fail_msg, writers_schema, readers_schema)
selected_writers_schema = writers_schema.schemas[index_of_schema]
-
+
# read data
return self.read_data(selected_writers_schema, readers_schema, decoder)
@@ -914,7 +919,7 @@ class DatumReader(object):
* if the reader's record schema has a field that contains a default value,
and writer's schema does not have a field with the same name, then the
reader should use the default value from its field.
- * if the reader's record schema has a field with no default value, and
+ * if the reader's record schema has a field with no default value, and
writer's schema does not have a field with the same name, then the
field's value is unset.
"""
@@ -933,7 +938,7 @@ class DatumReader(object):
if len(readers_fields_dict) > len(read_record):
writers_fields_dict = writers_schema.fields_dict
for field_name, field in readers_fields_dict.items():
- if not writers_fields_dict.has_key(field_name):
+ if field_name not in writers_fields_dict:
if field.has_default:
field_val = self._read_default_value(field.type, field.default)
read_record[field.name] = field_val
diff --git a/lang/py/src/avro/ipc.py b/lang/py/src/avro/ipc.py
index 8cbf07b..1bba4f7 100644
--- a/lang/py/src/avro/ipc.py
+++ b/lang/py/src/avro/ipc.py
@@ -1,3 +1,6 @@
+#!/usr/bin/env python
+
+##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -5,17 +8,19 @@
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
-#
+#
# https://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-"""
-Support for inter-process calls.
-"""
+
+"""Support for inter-process calls."""
+
+from __future__ import absolute_import, division, print_function
+
import httplib
from avro import io, protocol, schema
@@ -189,7 +194,7 @@ class BaseRequestor(object):
* a one-byte error flag boolean, followed by either:
o if the error flag is false,
the message response, serialized per the message's response schema.
- o if the error flag is true,
+ o if the error flag is true,
the error, serialized per the message's error union schema.
"""
# response metadata
@@ -267,11 +272,11 @@ class Responder(object):
buffer_encoder = io.BinaryEncoder(buffer_writer)
error = None
response_metadata = {}
-
+
try:
remote_protocol = self.process_handshake(buffer_decoder, buffer_encoder)
# handshake failure
- if remote_protocol is None:
+ if remote_protocol is None:
return buffer_writer.getvalue()
# read request using remote protocol
@@ -296,9 +301,9 @@ class Responder(object):
# perform server logic
try:
response = self.invoke(local_message, request)
- except AvroRemoteException, e:
+ except AvroRemoteException as e:
error = e
- except Exception, e:
+ except Exception as e:
error = AvroRemoteException(str(e))
# write response using local protocol
@@ -310,7 +315,7 @@ class Responder(object):
else:
writers_schema = local_message.errors
self.write_error(writers_schema, error, buffer_encoder)
- except schema.AvroException, e:
+ except schema.AvroException as e:
error = AvroRemoteException(str(e))
buffer_encoder = io.BinaryEncoder(StringIO())
META_WRITER.write(response_metadata, buffer_encoder)
diff --git a/lang/py/src/avro/protocol.py b/lang/py/src/avro/protocol.py
index 117401c..9b27a45 100644
--- a/lang/py/src/avro/protocol.py
+++ b/lang/py/src/avro/protocol.py
@@ -1,6 +1,3 @@
-#!/usr/bin/env python
-
-##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -21,10 +18,15 @@
from __future__ import absolute_import, division, print_function
-import hashlib
import json
-import avro.schema
+from avro import schema
+
+try:
+ from hashlib import md5
+except ImportError:
+ from md5 import md5
+
#
# Constants
@@ -37,7 +39,7 @@ VALID_TYPE_SCHEMA_TYPES = ('enum', 'record', 'error', 'fixed')
# Exceptions
#
-class ProtocolParseException(avro.schema.AvroException):
+class ProtocolParseException(schema.AvroException):
pass
#
@@ -49,7 +51,7 @@ class Protocol(object):
def _parse_types(self, types, type_names):
type_objects = []
for type in types:
- type_object = avro.schema.make_avsc_object(type, type_names)
+ type_object = schema.make_avsc_object(type, type_names)
if type_object.type not in VALID_TYPE_SCHEMA_TYPES:
fail_msg = 'Type %s not an enum, fixed, record, or error.' % type
raise ProtocolParseException(fail_msg)
@@ -92,7 +94,7 @@ class Protocol(object):
self._props = {}
self.set_prop('name', name)
- type_names = avro.schema.Names()
+ type_names = schema.Names()
if namespace is not None:
self.set_prop('namespace', namespace)
type_names.default_namespace = namespace
@@ -100,13 +102,13 @@ class Protocol(object):
self.set_prop('types', self._parse_types(types, type_names))
if messages is not None:
self.set_prop('messages', self._parse_messages(messages, type_names))
- self._md5 = hashlib.md5(str(self)).digest()
+ self._md5 = md5(str(self)).digest()
# read-only properties
name = property(lambda self: self.get_prop('name'))
namespace = property(lambda self: self.get_prop('namespace'))
fullname = property(lambda self:
- avro.schema.Name(self.name, self.namespace).fullname)
+ schema.Name(self.name, self.namespace).fullname)
types = property(lambda self: self.get_prop('types'))
types_dict = property(lambda self: dict([(type.name, type)
for type in self.types]))
@@ -123,7 +125,7 @@ class Protocol(object):
def to_json(self):
to_dump = {}
to_dump['protocol'] = self.name
- names = avro.schema.Names(default_namespace=self.namespace)
+ names = schema.Names(default_namespace=self.namespace)
if self.namespace:
to_dump['namespace'] = self.namespace
if self.types:
@@ -148,20 +150,20 @@ class Message(object):
if not isinstance(request, list):
fail_msg = 'Request property not a list: %s' % request
raise ProtocolParseException(fail_msg)
- return avro.schema.RecordSchema(None, None, request, names, 'request')
+ return schema.RecordSchema(None, None, request, names, 'request')
def _parse_response(self, response, names):
if isinstance(response, basestring) and names.has_name(response, None):
return names.get_name(response, None)
else:
- return avro.schema.make_avsc_object(response, names)
+ return schema.make_avsc_object(response, names)
def _parse_errors(self, errors, names):
if not isinstance(errors, list):
fail_msg = 'Errors property not a list: %s' % errors
raise ProtocolParseException(fail_msg)
errors_for_parsing = {'type': 'error_union', 'declared_errors': errors}
- return avro.schema.make_avsc_object(errors_for_parsing, names)
+ return schema.make_avsc_object(errors_for_parsing, names)
def __init__(self, name, request, response, errors=None, names=None):
self._name = name
@@ -190,7 +192,7 @@ class Message(object):
def to_json(self, names=None):
if names is None:
- names = avro.schema.Names()
+ names = schema.Names()
to_dump = {}
to_dump['request'] = self.request.to_json(names)
to_dump['response'] = self.response.to_json(names)
diff --git a/lang/py/src/avro/schema.py b/lang/py/src/avro/schema.py
index 822d76c..06c1aeb 100644
--- a/lang/py/src/avro/schema.py
+++ b/lang/py/src/avro/schema.py
@@ -1,3 +1,6 @@
+#!/usr/bin/env python
+
+##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -35,7 +38,10 @@ A schema may be one of:
Null.
"""
+from __future__ import absolute_import, division, print_function
+
import json
+import sys
from math import floor, log10
from avro import constants
@@ -231,11 +237,11 @@ class Names(object):
def has_name(self, name_attr, space_attr):
test = Name(name_attr, space_attr, self.default_namespace).fullname
- return self.names.has_key(test)
+ return test in self.names
def get_name(self, name_attr, space_attr):
test = Name(name_attr, space_attr, self.default_namespace).fullname
- if not self.names.has_key(test):
+ if test not in self.names:
return None
return self.names[test]
@@ -270,7 +276,7 @@ class Names(object):
if to_add.fullname in VALID_TYPES:
fail_msg = '%s is a reserved type name.' % to_add.fullname
raise SchemaParseException(fail_msg)
- elif self.names.has_key(to_add.fullname):
+ elif to_add.fullname in self.names:
fail_msg = 'The name "%s" is already in use.' % to_add.fullname
raise SchemaParseException(fail_msg)
@@ -377,7 +383,7 @@ class Field(object):
else:
try:
type_schema = make_avsc_object(type, names)
- except Exception, e:
+ except Exception as e:
fail_msg = 'Type property "%s" not a valid Avro schema: %s' % (type, e)
raise SchemaParseException(fail_msg)
self.set_prop('type', type_schema)
@@ -578,7 +584,7 @@ class ArraySchema(Schema):
else:
try:
items_schema = make_avsc_object(items, names)
- except SchemaParseException, e:
+ except SchemaParseException as e:
fail_msg = 'Items schema (%s) not a valid Avro schema: %s (known names: %s)' % (items, e, names.names.keys())
raise SchemaParseException(fail_msg)
@@ -652,7 +658,7 @@ class UnionSchema(Schema):
else:
try:
new_schema = make_avsc_object(schema, names)
- except Exception, e:
+ except Exception as e:
raise SchemaParseException('Union item must be a valid Avro schema: %s' % str(e))
# check the new schema
if (new_schema.type in VALID_TYPES and new_schema.type not in NAMED_TYPES
@@ -708,7 +714,7 @@ class RecordSchema(NamedSchema):
# null values can have a default value of None
has_default = False
default = None
- if field.has_key('default'):
+ if 'default' in field:
has_default = True
default = field.get('default')
@@ -978,10 +984,13 @@ def parse(json_string):
# parse the JSON
try:
json_data = json.loads(json_string)
- except Exception, e:
- import sys
- raise SchemaParseException('Error parsing JSON: %s, error = %s'
- % (json_string, e)), None, sys.exc_info()[2]
+ except Exception as e:
+ msg = 'Error parsing JSON: {}, error = {}'.format(json_string, e)
+ new_exception = SchemaParseException(msg)
+ traceback = sys.exc_info()[2]
+ if not hasattr(new_exception, 'with_traceback'):
+ raise (new_exception, None, traceback) # Python 2 syntax
+ raise new_exception.with_traceback(traceback)
# Initialize the names object
names = Names()
diff --git a/lang/py/src/avro/tether/__init__.py b/lang/py/src/avro/tether/__init__.py
index 0dbd3d8..c60edf9 100644
--- a/lang/py/src/avro/tether/__init__.py
+++ b/lang/py/src/avro/tether/__init__.py
@@ -1,4 +1,6 @@
-#
+#!/usr/bin/env python
+
+##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -15,12 +17,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#
-from .util import *
-from .tether_task import *
-from .tether_task_runner import *
+from __future__ import absolute_import, division, print_function
-__all__=util.__all__
-__all__+=tether_task.__all__
-__all__+=tether_task_runner.__all__
+from avro.tether.tether_task import HTTPRequestor, TaskType, TetherTask, inputProtocol, outputProtocol
+from avro.tether.tether_task_runner import TaskRunner
+from avro.tether.util import find_port
diff --git a/lang/py/src/avro/tether/tether_task.py b/lang/py/src/avro/tether/tether_task.py
index 23112a7..4e2004d 100644
--- a/lang/py/src/avro/tether/tether_task.py
+++ b/lang/py/src/avro/tether/tether_task.py
@@ -1,22 +1,23 @@
-"""
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-"""
-
-__all__=["TetherTask","TaskType","inputProtocol","outputProtocol","HTTPRequestor"]
+#!/usr/bin/env python
+
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import, division, print_function
import collections
import io as pyio
@@ -30,6 +31,8 @@ from StringIO import StringIO
from avro import io as avio
from avro import ipc, protocol, schema
+__all__ = ["TetherTask", "TaskType", "inputProtocol", "outputProtocol", "HTTPRequestor"]
+
# create protocol objects for the input and output protocols
# The build process should copy InputProtocol.avpr and OutputProtocol.avpr
# into the same directory as this module
diff --git a/lang/py/src/avro/tether/tether_task_runner.py b/lang/py/src/avro/tether/tether_task_runner.py
index b248ffd..64bee7b 100644
--- a/lang/py/src/avro/tether/tether_task_runner.py
+++ b/lang/py/src/avro/tether/tether_task_runner.py
@@ -1,30 +1,23 @@
-"""
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-"""
-
-__all__=["TaskRunner"]
-
-if __name__ == "__main__":
- # Relative imports don't work when being run directly
- from avro import tether
- from avro.tether import TetherTask, find_port, inputProtocol
-
-else:
- from . import TetherTask, find_port, inputProtocol
+#!/usr/bin/env python
+
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import, division, print_function
import logging
import sys
@@ -33,12 +26,16 @@ import traceback
import weakref
from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
+import avro.tether.tether_task
+import avro.tether.util
from avro import ipc
+__all__ = ["TaskRunner"]
+
class TaskRunnerResponder(ipc.Responder):
"""
- The responder for the thethered process
+ The responder for the tethered process
"""
def __init__(self,runner):
"""
@@ -46,7 +43,7 @@ class TaskRunnerResponder(ipc.Responder):
----------------------------------------------------------
runner - Instance of TaskRunner
"""
- ipc.Responder.__init__(self, inputProtocol)
+ ipc.Responder.__init__(self, avro.tether.tether_task.inputProtocol)
self.log=logging.getLogger("TaskRunnerResponder")
@@ -148,7 +145,7 @@ class TaskRunner(object):
self.log=logging.getLogger("TaskRunner:")
- if not(isinstance(task,TetherTask)):
+ if not(isinstance(task, avro.tether.tether_task.TetherTask)):
raise ValueError("task must be an instance of tether task")
self.task=task
@@ -172,7 +169,7 @@ class TaskRunner(object):
testing
"""
- port=find_port()
+ port = avro.tether.util.find_port()
address=("localhost",port)
@@ -212,7 +209,7 @@ if __name__ == '__main__':
logging.basicConfig(level=logging.INFO)
if (len(sys.argv)<=1):
- print "Error: tether_task_runner.__main__: Usage: tether_task_runner task_package.task_module.TaskClass"
+ print("Error: tether_task_runner.__main__: Usage: tether_task_runner task_package.task_module.TaskClass")
raise ValueError("Usage: tether_task_runner task_package.task_module.TaskClass")
fullcls=sys.argv[1]
diff --git a/lang/py/src/avro/tether/util.py b/lang/py/src/avro/tether/util.py
index cbeeef0..3d8ad3a 100644
--- a/lang/py/src/avro/tether/util.py
+++ b/lang/py/src/avro/tether/util.py
@@ -1,22 +1,23 @@
-"""
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-"""
+#!/usr/bin/env python
-__all__=["find_port"]
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import, division, print_function
import socket
diff --git a/lang/py/src/avro/timezones.py b/lang/py/src/avro/timezones.py
index a4985b4..a306f6d 100644
--- a/lang/py/src/avro/timezones.py
+++ b/lang/py/src/avro/timezones.py
@@ -1,3 +1,6 @@
+#!/usr/bin/env python
+
+##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -5,15 +8,17 @@
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
-#
+#
# https://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import, division, print_function
+
from datetime import datetime, timedelta, tzinfo
diff --git a/lang/py/src/avro/tool.py b/lang/py/src/avro/tool.py
index 6a92fee..3c0c228 100644
--- a/lang/py/src/avro/tool.py
+++ b/lang/py/src/avro/tool.py
@@ -1,4 +1,6 @@
-#! /usr/bin/env python
+#!/usr/bin/env python
+
+##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -6,20 +8,23 @@
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
-#
+#
# https://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
"""
Command-line tool
NOTE: The API for the command-line tool is experimental.
"""
+from __future__ import absolute_import, division, print_function
+
import sys
import threading
import urlparse
@@ -37,7 +42,7 @@ class GenericResponder(ipc.Responder):
def invoke(self, message, request):
if message.name == self.msg:
- print >> sys.stderr, "Message: %s Datum: %s" % (message.name, self.datum)
+ print("Message: %s Datum: %s" % (message.name, self.datum), file=sys.stderr)
# server will shut down after processing a single Avro request
global server_should_shutdown
server_should_shutdown = True
@@ -55,7 +60,7 @@ class GenericHandler(BaseHTTPRequestHandler):
resp_writer = ipc.FramedWriter(self.wfile)
resp_writer.write_framed_message(resp_body)
if server_should_shutdown:
- print >> sys.stderr, "Shutting down server."
+ print("Shutting down server.", file=sys.stderr)
quitter = threading.Thread(target=self.server.shutdown)
quitter.daemon = True
quitter.start()
@@ -68,10 +73,10 @@ def run_server(uri, proto, msg, datum):
server_should_shutdown = False
responder = GenericResponder(proto, msg, datum)
server = HTTPServer(server_addr, GenericHandler)
- print "Port: %s" % server.server_port
+ print("Port: %s" % server.server_port)
sys.stdout.flush()
server.allow_reuse_address = True
- print >> sys.stderr, "Starting server."
+ print("Starting server.", file=sys.stderr)
server.serve_forever()
def send_message(uri, proto, msg, datum):
@@ -79,7 +84,7 @@ def send_message(uri, proto, msg, datum):
client = ipc.HTTPTransceiver(url_obj.hostname, url_obj.port)
proto_json = file(proto, 'r').read()
requestor = ipc.Requestor(protocol.parse(proto_json), client)
- print requestor.request(msg, datum)
+ print(requestor.request(msg, datum))
def file_or_stdin(f):
if f == "-":
@@ -89,20 +94,20 @@ def file_or_stdin(f):
def main(args=sys.argv):
if len(args) == 1:
- print "Usage: %s [dump|rpcreceive|rpcsend]" % args[0]
+ print("Usage: %s [dump|rpcreceive|rpcsend]" % args[0])
return 1
if args[1] == "dump":
if len(args) != 3:
- print "Usage: %s dump input_file" % args[0]
+ print("Usage: %s dump input_file" % args[0])
return 1
for d in datafile.DataFileReader(file_or_stdin(args[2]), io.DatumReader()):
- print repr(d)
+ print(repr(d))
elif args[1] == "rpcreceive":
usage_str = "Usage: %s rpcreceive uri protocol_file " % args[0]
usage_str += "message_name (-data d | -file f)"
if len(args) not in [5, 7]:
- print usage_str
+ print(usage_str)
return 1
uri, proto, msg = args[2:5]
datum = None
@@ -111,19 +116,19 @@ def main(args=sys.argv):
reader = open(args[6], 'rb')
datum_reader = io.DatumReader()
dfr = datafile.DataFileReader(reader, datum_reader)
- datum = dfr.next()
+ datum = next(dfr)
elif args[5] == "-data":
- print "JSON Decoder not yet implemented."
+ print("JSON Decoder not yet implemented.")
return 1
else:
- print usage_str
+ print(usage_str)
return 1
run_server(uri, proto, msg, datum)
elif args[1] == "rpcsend":
usage_str = "Usage: %s rpcsend uri protocol_file " % args[0]
usage_str += "message_name (-data d | -file f)"
if len(args) not in [5, 7]:
- print usage_str
+ print(usage_str)
return 1
uri, proto, msg = args[2:5]
datum = None
@@ -132,15 +137,15 @@ def main(args=sys.argv):
reader = open(args[6], 'rb')
datum_reader = io.DatumReader()
dfr = datafile.DataFileReader(reader, datum_reader)
- datum = dfr.next()
+ datum = next(dfr)
elif args[5] == "-data":
- print "JSON Decoder not yet implemented."
+ print("JSON Decoder not yet implemented.")
return 1
else:
- print usage_str
+ print(usage_str)
return 1
send_message(uri, proto, msg, datum)
return 0
-
+
if __name__ == "__main__":
sys.exit(main(sys.argv))
diff --git a/lang/py/src/avro/txipc.py b/lang/py/src/avro/txipc.py
index 72d63a6..66ca726 100644
--- a/lang/py/src/avro/txipc.py
+++ b/lang/py/src/avro/txipc.py
@@ -1,5 +1,6 @@
#!/usr/bin/env python
+##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -15,10 +16,9 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-try:
- from cStringIO import StringIO
-except ImportError:
- from StringIO import StringIO
+
+from __future__ import absolute_import, division, print_function
+
from zope.interface import implements
from avro import io, ipc
@@ -29,6 +29,11 @@ from twisted.web.client import Agent
from twisted.web.http_headers import Headers
from twisted.web.iweb import IBodyProducer
+try:
+ from cStringIO import StringIO
+except ImportError:
+ from StringIO import StringIO
+
class TwistedRequestor(ipc.BaseRequestor):
"""A Twisted-compatible requestor. Returns a Deferred that will fire with the
diff --git a/lang/py/src/avro/__init__.py b/lang/py/test/__init__.py
similarity index 89%
copy from lang/py/src/avro/__init__.py
copy to lang/py/test/__init__.py
index 47d1295..a2a5bef 100644
--- a/lang/py/src/avro/__init__.py
+++ b/lang/py/test/__init__.py
@@ -1,3 +1,6 @@
+#!/usr/bin/env python
+
+##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -5,14 +8,11 @@
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
-#
+#
# https://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
-__all__ = ['schema', 'io', 'datafile', 'protocol', 'ipc', 'constants', 'timezones']
-
diff --git a/lang/py/test/av_bench.py b/lang/py/test/av_bench.py
index e90c987..1e6a05d 100644
--- a/lang/py/test/av_bench.py
+++ b/lang/py/test/av_bench.py
@@ -1,5 +1,6 @@
#!/usr/bin/env python
+##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,6 +17,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import, division, print_function
+
import sys
import time
from random import choice, randint, sample
@@ -72,5 +75,5 @@ def t(f, *args):
if __name__ == "__main__":
n = int(sys.argv[1])
- print "Write %0.4f" % t(write, n)
- print "Read %0.4f" % t(read)
+ print("Write %0.4f" % t(write, n))
+ print("Read %0.4f" % t(read))
diff --git a/lang/py/test/gen_interop_data.py b/lang/py/test/gen_interop_data.py
index 336434e..13bf86c 100644
--- a/lang/py/test/gen_interop_data.py
+++ b/lang/py/test/gen_interop_data.py
@@ -1,5 +1,6 @@
#!/usr/bin/env python
+##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -13,15 +14,33 @@
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
# See the License for the specific language governing permissions and
# limitations under the License.
+
+from __future__ import absolute_import, division, print_function
+
+import os
import sys
from avro import datafile, io, schema
+CODECS_TO_VALIDATE = ('null', 'deflate')
+
+try:
+ import snappy
+ CODECS_TO_VALIDATE += ('snappy',)
+except ImportError:
+ print('Snappy not present, will skip generating it.')
+try:
+ import zstandard
+ CODECS_TO_VALIDATE += ('zstandard',)
+except ImportError:
+ print('Zstandard not present, will skip generating it.')
+
DATUM = {
'intField': 12,
- 'longField': 15234324L,
+ 'longField': 15234324,
'stringField': unicode('hey'),
'boolField': True,
'floatField': 1234.0,
@@ -37,10 +56,15 @@ DATUM = {
}
if __name__ == "__main__":
- interop_schema = schema.parse(open(sys.argv[1], 'r').read())
- writer = open(sys.argv[2], 'wb')
- datum_writer = io.DatumWriter()
- # NB: not using compression
- dfw = datafile.DataFileWriter(writer, datum_writer, interop_schema)
- dfw.append(DATUM)
- dfw.close()
+ for codec in CODECS_TO_VALIDATE:
+ interop_schema = schema.parse(open(sys.argv[1], 'r').read())
+ filename = sys.argv[2]
+ if codec != 'null':
+ base, ext = os.path.splitext(filename)
+ filename = base + "_" + codec + ext
+ writer = open(filename, 'wb')
+ datum_writer = io.DatumWriter()
+ # NB: not using compression
+ dfw = datafile.DataFileWriter(writer, datum_writer, interop_schema, codec=codec)
+ dfw.append(DATUM)
+ dfw.close()
diff --git a/lang/py/test/mock_tether_parent.py b/lang/py/test/mock_tether_parent.py
index c82e249..88d84dd 100644
--- a/lang/py/test/mock_tether_parent.py
+++ b/lang/py/test/mock_tether_parent.py
@@ -1,3 +1,6 @@
+#!/usr/bin/env python
+
+##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -14,45 +17,36 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import, division, print_function
+
import socket
import sys
from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
+import avro.tether.tether_task
+import avro.tether.util
import set_avro_test_path
-from avro import ipc, protocol, tether
-
-
-def find_port():
- """
- Return an unbound port
- """
- s=socket.socket()
- s.bind(("127.0.0.1",0))
-
- port=s.getsockname()[1]
- s.close()
-
- return port
+from avro import ipc, protocol
-SERVER_ADDRESS = ('localhost', find_port())
+SERVER_ADDRESS = ('localhost', avro.tether.util.find_port())
class MockParentResponder(ipc.Responder):
"""
The responder for the mocked parent
"""
def __init__(self):
- ipc.Responder.__init__(self, tether.outputProtocol)
+ ipc.Responder.__init__(self, avro.tether.tether_task.outputProtocol)
def invoke(self, message, request):
if message.name=='configure':
- print "MockParentResponder: Recieved 'configure': inputPort={0}".format(request["port"])
+ print("MockParentResponder: Recieved 'configure': inputPort={0}".format(request["port"]))
elif message.name=='status':
- print "MockParentResponder: Recieved 'status': message={0}".format(request["message"])
+ print("MockParentResponder: Recieved 'status': message={0}".format(request["message"]))
elif message.name=='fail':
- print "MockParentResponder: Recieved 'fail': message={0}".format(request["message"])
+ print("MockParentResponder: Recieved 'fail': message={0}".format(request["message"]))
else:
- print "MockParentResponder: Recieved {0}".format(message.name)
+ print("MockParentResponder: Recieved {0}".format(message.name))
# flush the output so it shows up in the parent process
sys.stdout.flush()
@@ -85,7 +79,7 @@ if __name__ == '__main__':
raise ValueError("Usage: mock_tether_parent start_server port")
SERVER_ADDRESS=(SERVER_ADDRESS[0],port)
- print "mock_tether_parent: Launching Server on Port: {0}".format(SERVER_ADDRESS[1])
+ print("mock_tether_parent: Launching Server on Port: {0}".format(SERVER_ADDRESS[1]))
# flush the output so it shows up in the parent process
sys.stdout.flush()
diff --git a/lang/py/test/sample_http_client.py b/lang/py/test/sample_http_client.py
index 62c91fd..02b8421 100644
--- a/lang/py/test/sample_http_client.py
+++ b/lang/py/test/sample_http_client.py
@@ -1,5 +1,6 @@
#!/usr/bin/env python
+##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -13,8 +14,12 @@
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
# See the License for the specific language governing permissions and
# limitations under the License.
+
+from __future__ import absolute_import, division, print_function
+
import sys
from avro import ipc, protocol
@@ -78,7 +83,7 @@ if __name__ == '__main__':
# build the parameters for the request
params = {}
params['message'] = message
-
+
# send the requests and print the result
for msg_count in range(num_messages):
requestor = make_requestor(SERVER_HOST, SERVER_PORT, MAIL_PROTOCOL)
diff --git a/lang/py/test/sample_http_server.py b/lang/py/test/sample_http_server.py
index e412ab5..c680afb 100644
--- a/lang/py/test/sample_http_server.py
+++ b/lang/py/test/sample_http_server.py
@@ -1,5 +1,6 @@
#!/usr/bin/env python
+##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,6 +17,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import, division, print_function
+
from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
from avro import ipc, protocol
diff --git a/lang/py/test/set_avro_test_path.py b/lang/py/test/set_avro_test_path.py
index 8e47faf..fd395da 100644
--- a/lang/py/test/set_avro_test_path.py
+++ b/lang/py/test/set_avro_test_path.py
@@ -1,3 +1,6 @@
+#!/usr/bin/env python
+
+##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -13,6 +16,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
"""
Module adjusts the path PYTHONPATH so the unittests
will work even if an egg for AVRO is already installed.
@@ -28,6 +32,9 @@ being built. To work around this the unittests import this module before
importing AVRO. This module in turn adjusts the python path so that the test
build of AVRO is higher on the path then any installed eggs.
"""
+
+from __future__ import absolute_import, division, print_function
+
import os
import sys
diff --git a/lang/py/test/test_datafile.py b/lang/py/test/test_datafile.py
index 2b7061c..bceb071 100644
--- a/lang/py/test/test_datafile.py
+++ b/lang/py/test/test_datafile.py
@@ -1,3 +1,6 @@
+#!/usr/bin/env python
+
+##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -5,14 +8,17 @@
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
-#
+#
# https://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
+from __future__ import absolute_import, division, print_function
+
import os
import unittest
@@ -56,25 +62,30 @@ try:
import snappy
CODECS_TO_VALIDATE += ('snappy',)
except ImportError:
- print 'Snappy not present, will skip testing it.'
+ print('Snappy not present, will skip testing it.')
+try:
+ import zstandard
+ CODECS_TO_VALIDATE += ('zstandard',)
+except ImportError:
+ print('Zstandard not present, will skip testing it.')
# TODO(hammer): clean up written files with ant, not os.remove
class TestDataFile(unittest.TestCase):
def test_round_trip(self):
- print ''
- print 'TEST ROUND TRIP'
- print '==============='
- print ''
+ print('')
+ print('TEST ROUND TRIP')
+ print('===============')
+ print('')
correct = 0
for i, (example_schema, datum) in enumerate(SCHEMAS_TO_VALIDATE):
for codec in CODECS_TO_VALIDATE:
- print ''
- print 'SCHEMA NUMBER %d' % (i + 1)
- print '================'
- print ''
- print 'Schema: %s' % example_schema
- print 'Datum: %s' % datum
- print 'Codec: %s' % codec
+ print('')
+ print('SCHEMA NUMBER %d' % (i + 1))
+ print('================')
+ print('')
+ print('Schema: %s' % example_schema)
+ print('Datum: %s' % datum)
+ print('Codec: %s' % codec)
# write data in binary to file 10 times
writer = open(FILENAME, 'wb')
@@ -93,30 +104,30 @@ class TestDataFile(unittest.TestCase):
for datum in dfr:
round_trip_data.append(datum)
- print 'Round Trip Data: %s' % round_trip_data
- print 'Round Trip Data Length: %d' % len(round_trip_data)
+ print('Round Trip Data: %s' % round_trip_data)
+ print('Round Trip Data Length: %d' % len(round_trip_data))
is_correct = [datum] * 10 == round_trip_data
if is_correct: correct += 1
- print 'Correct Round Trip: %s' % is_correct
- print ''
+ print('Correct Round Trip: %s' % is_correct)
+ print('')
os.remove(FILENAME)
self.assertEquals(correct, len(CODECS_TO_VALIDATE)*len(SCHEMAS_TO_VALIDATE))
def test_append(self):
- print ''
- print 'TEST APPEND'
- print '==========='
- print ''
+ print('')
+ print('TEST APPEND')
+ print('===========')
+ print('')
correct = 0
for i, (example_schema, datum) in enumerate(SCHEMAS_TO_VALIDATE):
for codec in CODECS_TO_VALIDATE:
- print ''
- print 'SCHEMA NUMBER %d' % (i + 1)
- print '================'
- print ''
- print 'Schema: %s' % example_schema
- print 'Datum: %s' % datum
- print 'Codec: %s' % codec
+ print('')
+ print('SCHEMA NUMBER %d' % (i + 1))
+ print('================')
+ print('')
+ print('Schema: %s' % example_schema)
+ print('Datum: %s' % datum)
+ print('Codec: %s' % codec)
# write data in binary to file once
writer = open(FILENAME, 'wb')
@@ -141,12 +152,12 @@ class TestDataFile(unittest.TestCase):
for datum in dfr:
appended_data.append(datum)
- print 'Appended Data: %s' % appended_data
- print 'Appended Data Length: %d' % len(appended_data)
+ print('Appended Data: %s' % appended_data)
+ print('Appended Data Length: %d' % len(appended_data))
is_correct = [datum] * 10 == appended_data
if is_correct: correct += 1
- print 'Correct Appended: %s' % is_correct
- print ''
+ print('Correct Appended: %s' % is_correct)
+ print('')
os.remove(FILENAME)
self.assertEquals(correct, len(CODECS_TO_VALIDATE)*len(SCHEMAS_TO_VALIDATE))
diff --git a/lang/py/test/test_datafile_interop.py b/lang/py/test/test_datafile_interop.py
index ee02f99..329b9a1 100644
--- a/lang/py/test/test_datafile_interop.py
+++ b/lang/py/test/test_datafile_interop.py
@@ -1,3 +1,6 @@
+#!/usr/bin/env python
+
+##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -5,14 +8,17 @@
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
-#
+#
# https://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
+from __future__ import absolute_import, division, print_function
+
import os
import unittest
@@ -22,20 +28,27 @@ from avro import datafile, io
class TestDataFileInterop(unittest.TestCase):
def test_interop(self):
- print ''
- print 'TEST INTEROP'
- print '============'
- print ''
+ print()
+ print('TEST INTEROP')
+ print('============')
+ print()
for f in os.listdir('@INTEROP_DATA_DIR@'):
- print 'READING %s' % f
- print ''
-
- # read data in binary from file
- reader = open(os.path.join('@INTEROP_DATA_DIR@', f), 'rb')
- datum_reader = io.DatumReader()
- dfr = datafile.DataFileReader(reader, datum_reader)
- for datum in dfr:
- assert datum is not None
+ base_ext = os.path.splitext(os.path.basename(f))[0].split('_', 1)
+ if len(base_ext) < 2 or base_ext[1] in datafile.VALID_CODECS:
+ print('READING %s' % f)
+ print('')
+
+ # read data in binary from file
+ reader = open(os.path.join('@INTEROP_DATA_DIR@', f), 'rb')
+ datum_reader = io.DatumReader()
+ dfr = datafile.DataFileReader(reader, datum_reader)
+ i = 0
+ for i, datum in enumerate(dfr, 1):
+ assert datum is not None
+ assert i > 0
+ else:
+ print('SKIPPING %s due to an unsupported codec' % f)
+ print('')
if __name__ == '__main__':
unittest.main()
diff --git a/lang/py/test/test_io.py b/lang/py/test/test_io.py
index 533aa40..2d734a5 100644
--- a/lang/py/test/test_io.py
+++ b/lang/py/test/test_io.py
@@ -1,3 +1,6 @@
+#!/usr/bin/env python
+
+##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -5,14 +8,17 @@
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
-#
+#
# https://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
+from __future__ import absolute_import, division, print_function
+
import datetime
import unittest
from binascii import hexlify
@@ -27,7 +33,6 @@ except ImportError:
from StringIO import StringIO
-
SCHEMAS_TO_VALIDATE = (
('"null"', None),
('"boolean"', True),
@@ -112,7 +117,7 @@ DEFAULT_VALUE_EXAMPLES = (
('"string"', '"foo"', u'foo'),
('"bytes"', '"\u00FF\u00FF"', u'\xff\xff'),
('"int"', '5', 5),
- ('"long"', '5', 5L),
+ ('"long"', '5', 5),
('"float"', '1.1', 1.1),
('"double"', '1.1', 1.1),
('{"type": "fixed", "name": "F", "size": 2}', '"\u00FF\u00FF"', u'\xff\xff'),
@@ -148,10 +153,10 @@ def avro_hexlify(reader):
return ' '.join(bytes)
def print_test_name(test_name):
- print ''
- print test_name
- print '=' * len(test_name)
- print ''
+ print('')
+ print(test_name)
+ print('=' * len(test_name))
+ print('')
def write_datum(datum, writers_schema):
writer = StringIO()
@@ -170,17 +175,17 @@ def check_binary_encoding(number_type):
print_test_name('TEST BINARY %s ENCODING' % number_type.upper())
correct = 0
for datum, hex_encoding in BINARY_ENCODINGS:
- print 'Datum: %d' % datum
- print 'Correct Encoding: %s' % hex_encoding
+ print('Datum: %d' % datum)
+ print('Correct Encoding: %s' % hex_encoding)
writers_schema = schema.parse('"%s"' % number_type.lower())
writer, encoder, datum_writer = write_datum(datum, writers_schema)
writer.seek(0)
hex_val = avro_hexlify(writer)
- print 'Read Encoding: %s' % hex_val
+ print('Read Encoding: %s' % hex_val)
if hex_encoding == hex_val: correct += 1
- print ''
+ print('')
return correct
def check_skip_number(number_type):
@@ -188,7 +193,7 @@ def check_skip_number(number_type):
correct = 0
for value_to_skip, hex_encoding in BINARY_ENCODINGS:
VALUE_TO_READ = 6253
- print 'Value to Skip: %d' % value_to_skip
+ print('Value to Skip: %d' % value_to_skip)
# write the value to skip and a known value
writers_schema = schema.parse('"%s"' % number_type.lower())
@@ -204,11 +209,11 @@ def check_skip_number(number_type):
datum_reader = io.DatumReader(writers_schema)
read_value = datum_reader.read(decoder)
- print 'Read Value: %d' % read_value
+ print('Read Value: %d' % read_value)
if read_value == VALUE_TO_READ: correct += 1
- print ''
+ print('')
return correct
-
+
class TestIO(unittest.TestCase):
#
# BASIC FUNCTIONALITY
@@ -218,10 +223,10 @@ class TestIO(unittest.TestCase):
print_test_name('TEST VALIDATE')
passed = 0
for example_schema, datum in SCHEMAS_TO_VALIDATE:
- print 'Schema: %s' % example_schema
- print 'Datum: %s' % datum
+ print('Schema: %s' % example_schema)
+ print('Datum: %s' % datum)
validated = io.validate(schema.parse(example_schema), datum)
- print 'Valid: %s' % validated
+ print('Valid: %s' % validated)
if validated: passed += 1
self.assertEquals(passed, len(SCHEMAS_TO_VALIDATE))
@@ -229,14 +234,14 @@ class TestIO(unittest.TestCase):
print_test_name('TEST ROUND TRIP')
correct = 0
for example_schema, datum in SCHEMAS_TO_VALIDATE:
- print 'Schema: %s' % example_schema
- print 'Datum: %s' % datum
+ print('Schema: %s' % example_schema)
+ print('Datum: %s' % datum)
writers_schema = schema.parse(example_schema)
writer, encoder, datum_writer = write_datum(datum, writers_schema)
round_trip_datum = read_datum(writer, writers_schema)
- print 'Round Trip Datum: %s' % round_trip_datum
+ print('Round Trip Datum: %s' % round_trip_datum)
if isinstance(round_trip_datum, Decimal):
round_trip_datum = round_trip_datum.to_eng_string()
datum = str(datum)
@@ -283,8 +288,8 @@ class TestIO(unittest.TestCase):
readers_schema = schema.parse(rs)
writer, enc, dw = write_datum(datum_to_write, writers_schema)
datum_read = read_datum(writer, writers_schema, readers_schema)
- print 'Writer: %s Reader: %s' % (writers_schema, readers_schema)
- print 'Datum Read: %s' % datum_read
+ print('Writer: %s Reader: %s' % (writers_schema, readers_schema))
+ print('Datum Read: %s' % datum_read)
if datum_read != datum_to_write: incorrect += 1
self.assertEquals(incorrect, 0)
@@ -320,7 +325,7 @@ class TestIO(unittest.TestCase):
writer, encoder, datum_writer = write_datum(datum_to_write, writers_schema)
datum_read = read_datum(writer, writers_schema, readers_schema)
- print 'Datum Read: %s' % datum_read
+ print('Datum Read: %s' % datum_read)
if datum_to_read == datum_read: correct += 1
self.assertEquals(correct, len(DEFAULT_VALUE_EXAMPLES))
@@ -352,7 +357,7 @@ class TestIO(unittest.TestCase):
writer, encoder, datum_writer = write_datum(datum_to_write, writers_schema)
datum_read = read_datum(writer, writers_schema, readers_schema)
- print 'Datum Read: %s' % datum_read
+ print('Datum Read: %s' % datum_read)
self.assertEquals(datum_to_read, datum_read)
def test_field_order(self):
@@ -368,7 +373,7 @@ class TestIO(unittest.TestCase):
writer, encoder, datum_writer = write_datum(datum_to_write, writers_schema)
datum_read = read_datum(writer, writers_schema, readers_schema)
- print 'Datum Read: %s' % datum_read
+ print('Datum Read: %s' % datum_read)
self.assertEquals(datum_to_read, datum_read)
def test_type_exception(self):
diff --git a/lang/py/test/test_ipc.py b/lang/py/test/test_ipc.py
index 575a0c9..bc9bd21 100644
--- a/lang/py/test/test_ipc.py
+++ b/lang/py/test/test_ipc.py
@@ -1,3 +1,6 @@
+#!/usr/bin/env python
+
+##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -13,10 +16,14 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
"""
There are currently no IPC tests within python, in part because there are no
servers yet available.
"""
+
+from __future__ import absolute_import, division, print_function
+
import unittest
import set_avro_test_path
diff --git a/lang/py/test/test_script.py b/lang/py/test/test_script.py
index 214fc15..bd0cb4d 100644
--- a/lang/py/test/test_script.py
+++ b/lang/py/test/test_script.py
@@ -1,3 +1,6 @@
+#!/usr/bin/env python
+
+##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -5,15 +8,17 @@
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
-#
+#
# https://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import, division, print_function
+
import csv
import json
import unittest
@@ -125,7 +130,7 @@ class TestCat(unittest.TestCase):
def test_json_pretty(self):
out = self._run("--format", "json-pretty", "-n", "1", raw=1)
- assert out.strip() == _JSON_PRETTY.strip()
+ self.assertEqual(out.strip(), _JSON_PRETTY.strip())
def test_version(self):
check_output([SCRIPT, "cat", "--version"])
diff --git a/lang/py/test/test_tether_task.py b/lang/py/test/test_tether_task.py
index 9933070..85ed9cb 100644
--- a/lang/py/test/test_tether_task.py
+++ b/lang/py/test/test_tether_task.py
@@ -1,3 +1,6 @@
+#!/usr/bin/env python
+
+##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -14,15 +17,22 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
+from __future__ import absolute_import, division, print_function
import os
+import StringIO
import subprocess
import sys
import time
import unittest
+import avro.tether.tether_task
+import avro.tether.util
+import mock_tether_parent
import set_avro_test_path
+from avro import io as avio
+from avro import schema, tether
+from word_count_task import WordCountTask
class TestTetherTask(unittest.TestCase):
@@ -34,15 +44,6 @@ class TestTetherTask(unittest.TestCase):
Test that the thether_task is working. We run the mock_tether_parent in a separate
subprocess
"""
- from avro import tether
- from avro import io as avio
- from avro import schema
- from avro.tether import HTTPRequestor,inputProtocol, find_port
-
- import StringIO
- import mock_tether_parent
- from word_count_task import WordCountTask
-
task=WordCountTask()
proc=None
@@ -51,13 +52,13 @@ class TestTetherTask(unittest.TestCase):
# env["AVRO_TETHER_OUTPUT_PORT"]=output_port
env=dict()
env["PYTHONPATH"]=':'.join(sys.path)
- server_port=find_port()
+ server_port = avro.tether.util.find_port()
pyfile=mock_tether_parent.__file__
proc=subprocess.Popen(["python", pyfile,"start_server","{0}".format(server_port)])
- input_port=find_port()
+ input_port = avro.tether.util.find_port()
- print "Mock server started process pid={0}".format(proc.pid)
+ print("Mock server started process pid={0}".format(proc.pid))
# Possible race condition? open tries to connect to the subprocess before the subprocess is fully started
# so we give the subprocess time to start up
time.sleep(1)
@@ -68,7 +69,11 @@ class TestTetherTask(unittest.TestCase):
#***************************************************************
# Test the mapper
- task.configure(tether.TaskType.MAP,str(task.inschema),str(task.midschema))
+ task.configure(
+ avro.tether.tether_task.TaskType.MAP,
+ str(task.inschema),
+ str(task.midschema)
+ )
# Serialize some data so we can send it to the input function
datum="This is a line of text"
@@ -84,7 +89,11 @@ class TestTetherTask(unittest.TestCase):
task.input(data,1)
# Test the reducer
- task.configure(tether.TaskType.REDUCE,str(task.midschema),str(task.outschema))
+ task.configure(
+ avro.tether.tether_task.TaskType.REDUCE,
+ str(task.midschema),
+ str(task.outschema)
+ )
# Serialize some data so we can send it to the input function
datum={"key":"word","value":2}
diff --git a/lang/py/test/test_tether_task_runner.py b/lang/py/test/test_tether_task_runner.py
index 3832dbe..985eb3c 100644
--- a/lang/py/test/test_tether_task_runner.py
+++ b/lang/py/test/test_tether_task_runner.py
@@ -1,3 +1,6 @@
+#!/usr/bin/env python
+
+##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -14,28 +17,29 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import, division, print_function
+
+import logging
import os
+import StringIO
import subprocess
import sys
import time
import unittest
+import avro.tether.tether_task
+import avro.tether.tether_task_runner
+import avro.tether.util
+import mock_tether_parent
import set_avro_test_path
+from avro import io as avio
+from word_count_task import WordCountTask
class TestTetherTaskRunner(unittest.TestCase):
- """ unit test for a tethered task runner.
- """
+ """unit test for a tethered task runner."""
def test1(self):
- from word_count_task import WordCountTask
- from avro.tether import TaskRunner, find_port,HTTPRequestor,inputProtocol, TaskType
- from avro import io as avio
- import mock_tether_parent
- import subprocess
- import StringIO
- import logging
-
# set the logging level to debug so that debug messages are printed
logging.basicConfig(level=logging.DEBUG)
@@ -44,30 +48,34 @@ class TestTetherTaskRunner(unittest.TestCase):
# launch the server in a separate process
env=dict()
env["PYTHONPATH"]=':'.join(sys.path)
- parent_port=find_port()
+ parent_port = avro.tether.util.find_port()
pyfile=mock_tether_parent.__file__
proc=subprocess.Popen(["python", pyfile,"start_server","{0}".format(parent_port)])
- input_port=find_port()
+ input_port = avro.tether.util.find_port()
- print "Mock server started process pid={0}".format(proc.pid)
+ print("Mock server started process pid={0}".format(proc.pid))
# Possible race condition? open tries to connect to the subprocess before the subprocess is fully started
# so we give the subprocess time to start up
time.sleep(1)
- runner=TaskRunner(WordCountTask())
+ runner = avro.tether.tether_task_runner.TaskRunner(WordCountTask())
runner.start(outputport=parent_port,join=False)
- # Test sending various messages to the server and ensuring they are
- # processed correctly
- requestor=HTTPRequestor("localhost",runner.server.server_address[1],inputProtocol)
+ # Test sending various messages to the server and ensuring they are processed correctly
+ requestor = avro.tether.tether_task.HTTPRequestor(
+ "localhost", runner.server.server_address[1], avro.tether.tether_task.inputProtocol)
# TODO: We should validate that open worked by grabbing the STDOUT of the subproces
# and ensuring that it outputted the correct message.
# Test the mapper
- requestor.request("configure",{"taskType":TaskType.MAP,"inSchema":str(runner.task.inschema),"outSchema":str(runner.task.midschema)})
+ requestor.request("configure", {
+ "taskType": avro.tether.tether_task.TaskType.MAP,
+ "inSchema": str(runner.task.inschema),
+ "outSchema": str(runner.task.midschema)
+ })
# Serialize some data so we can send it to the input function
datum="This is a line of text"
@@ -83,8 +91,12 @@ class TestTetherTaskRunner(unittest.TestCase):
# Call input to simulate calling map
requestor.request("input",{"data":data,"count":1})
- #Test the reducer
- requestor.request("configure",{"taskType":TaskType.REDUCE,"inSchema":str(runner.task.midschema),"outSchema":str(runner.task.outschema)})
+ # Test the reducer
+ requestor.request("configure", {
+ "taskType": avro.tether.tether_task.TaskType.REDUCE,
+ "inSchema": str(runner.task.midschema),
+ "outSchema": str(runner.task.outschema)}
+ )
#Serialize some data so we can send it to the input function
datum={"key":"word","value":2}
@@ -133,15 +145,6 @@ class TestTetherTaskRunner(unittest.TestCase):
as our main script everything works as expected. We do this by using subprocess to run it
in a separate thread.
"""
- from word_count_task import WordCountTask
- from avro.tether import TaskRunner, find_port,HTTPRequestor,inputProtocol, TaskType
- from avro.tether import tether_task_runner
- from avro import io as avio
- import mock_tether_parent
- import subprocess
- import StringIO
-
-
proc=None
runnerproc=None
@@ -149,7 +152,7 @@ class TestTetherTaskRunner(unittest.TestCase):
#launch the server in a separate process
env=dict()
env["PYTHONPATH"]=':'.join(sys.path)
- parent_port=find_port()
+ parent_port = avro.tether.util.find_port()
pyfile=mock_tether_parent.__file__
proc=subprocess.Popen(["python", pyfile,"start_server","{0}".format(parent_port)])
@@ -164,14 +167,14 @@ class TestTetherTaskRunner(unittest.TestCase):
env={"AVRO_TETHER_OUTPUT_PORT":"{0}".format(parent_port)}
env["PYTHONPATH"]=':'.join(sys.path)
- runnerproc=subprocess.Popen(["python",tether_task_runner.__file__,"word_count_task.WordCountTask"],env=env)
+ runnerproc = subprocess.Popen(["python", avro.tether.tether_task_runner.__file__, "word_count_task.WordCountTask"],env=env)
#possible race condition wait for the process to start
time.sleep(1)
- print "Mock server started process pid={0}".format(proc.pid)
+ print("Mock server started process pid={0}".format(proc.pid))
#Possible race condition? open tries to connect to the subprocess before the subprocess is fully started
#so we give the subprocess time to start up
time.sleep(1)
diff --git a/lang/py/test/test_tether_word_count.py b/lang/py/test/test_tether_word_count.py
index d2f1858..8c3fb08 100644
--- a/lang/py/test/test_tether_word_count.py
+++ b/lang/py/test/test_tether_word_count.py
@@ -1,3 +1,6 @@
+#!/usr/bin/env python
+
+##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -14,6 +17,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import, division, print_function
+
import inspect
import os
import subprocess
@@ -25,8 +30,7 @@ import set_avro_test_path
class TestTetherWordCount(unittest.TestCase):
- """ unittest for a python tethered map-reduce job.
- """
+ """unittest for a python tethered map-reduce job."""
def _write_lines(self,lines,fname):
"""
@@ -72,7 +76,7 @@ class TestTetherWordCount(unittest.TestCase):
words=line.split()
for w in words:
- if not(counts.has_key(w.strip())):
+ if not(w.strip() in counts):
counts[w.strip()]=0
counts[w.strip()]=counts[w.strip()]+1
@@ -92,7 +96,6 @@ class TestTetherWordCount(unittest.TestCase):
import avro
import subprocess
- import StringIO
import shutil
import tempfile
import inspect
@@ -182,11 +185,11 @@ python -m avro.tether.tether_task_runner word_count_task.WordCountTask
exhf.close()
# make it world executable
- os.chmod(exfile,0755)
+ os.chmod(exfile,0o755)
args.extend(["--program",exfile])
- print "Command:\n\t{0}".format(" ".join(args))
+ print("Command:\n\t{0}".format(" ".join(args)))
proc=subprocess.Popen(args)
diff --git a/lang/py/test/txsample_http_client.py b/lang/py/test/txsample_http_client.py
index dba4ade..28c2c28 100644
--- a/lang/py/test/txsample_http_client.py
+++ b/lang/py/test/txsample_http_client.py
@@ -1,5 +1,6 @@
#!/usr/bin/env python
+##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -13,8 +14,12 @@
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
# See the License for the specific language governing permissions and
# limitations under the License.
+
+from __future__ import absolute_import, division, print_function
+
import sys
from avro import protocol, txipc
diff --git a/lang/py/test/txsample_http_server.py b/lang/py/test/txsample_http_server.py
index 604ef54..fafaecd 100644
--- a/lang/py/test/txsample_http_server.py
+++ b/lang/py/test/txsample_http_server.py
@@ -1,5 +1,6 @@
#!/usr/bin/env python
+##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,6 +17,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import, division, print_function
+
from avro import ipc, protocol, txipc
from twisted.internet import reactor
from twisted.web import server
diff --git a/lang/py/test/word_count_task.py b/lang/py/test/word_count_task.py
index 24f2da2..8181340 100644
--- a/lang/py/test/word_count_task.py
+++ b/lang/py/test/word_count_task.py
@@ -1,33 +1,36 @@
-"""
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-"""
-
-__all__=["WordCountTask"]
+#!/usr/bin/env python
+
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import, division, print_function
import logging
-from avro.tether import TetherTask
+import avro.tether.tether_task
+
+__all__ = ["WordCountTask"]
#TODO::Make the logging level a parameter we can set
#logging.basicConfig(level=logging.INFO)
-class WordCountTask(TetherTask):
+class WordCountTask(avro.tether.tether_task.TetherTask):
"""
- Implements the mappper and reducer for the word count example
+ Implements the mapper and reducer for the word count example
"""
def __init__(self):
@@ -40,7 +43,7 @@ class WordCountTask(TetherTask):
{"name":"value","type":"long","order":"ignore"}]
}"""
outschema=midschema
- TetherTask.__init__(self,inschema,midschema,outschema)
+ avro.tether.tether_task.TetherTask.__init__(self, inschema, midschema, outschema)
#keep track of the partial sums of the counts
[avro] 14/23: AVRO-2580: Enforce Logical Type and Literal Type
Match (#668)
Posted by rs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rskraba pushed a commit to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/avro.git
commit e77241990490b8cd06df4dd7ea02d59535fab745
Author: Michael A. Smith <mi...@smith-li.com>
AuthorDate: Fri Oct 25 16:51:47 2019 -0400
AVRO-2580: Enforce Logical Type and Literal Type Match (#668)
* AVRO-2580: Refactor Schema.parse Test
* AVRO-2580: Rewrite Tests to Highlight Bug
* AVRO-2580: Require Logical Type to Match Literal Type
* AVRO-2580: Refactor Schema Tests
1. Enable showing multiple failures in a single run.
2. Use JSON to format test schema when possible.
---
lang/py/src/avro/schema.py | 61 ++--
lang/py/test/test_schema.py | 775 +++++++++++++++++---------------------------
2 files changed, 330 insertions(+), 506 deletions(-)
diff --git a/lang/py/src/avro/schema.py b/lang/py/src/avro/schema.py
index 776d6e1..822d76c 100644
--- a/lang/py/src/avro/schema.py
+++ b/lang/py/src/avro/schema.py
@@ -876,15 +876,36 @@ class TimestampMicrosSchema(LogicalSchema, PrimitiveSchema):
#
# Module Methods
#
-def get_other_props(all_props,reserved_props):
+def get_other_props(all_props, reserved_props):
"""
Retrieve the non-reserved properties from a dictionary of properties
@args reserved_props: The set of reserved properties to exclude
"""
if callable(getattr(all_props, 'items', None)):
- return dict([(k,v) for (k,v) in all_props.items() if k not in
- reserved_props ])
-
+ return {k: v for k, v in all_props.items() if k not in reserved_props}
+
+def make_bytes_decimal_schema(other_props):
+ """Make a BytesDecimalSchema from just other_props."""
+ return BytesDecimalSchema(other_props.get('precision'), other_props.get('scale', 0))
+
+def make_logical_schema(logical_type, type_, other_props):
+ """Map the logical types to the appropriate literal type and schema class."""
+ logical_types = {
+ constants.DATE: ('int', DateSchema),
+ # Fixed decimal schema is handled before we get here.
+ constants.DECIMAL: ('bytes', make_bytes_decimal_schema),
+ constants.TIMESTAMP_MICROS: ('long', TimestampMicrosSchema),
+ constants.TIMESTAMP_MILLIS: ('long', TimestampMillisSchema),
+ constants.TIME_MICROS: ('long', TimeMicrosSchema),
+ constants.TIME_MILLIS: ('int', TimeMillisSchema),
+ }
+ try:
+ literal_type, schema_type = logical_types[logical_type]
+ except KeyError:
+ raise SchemaParseException("Currently does not support {} logical type".format(logical_type))
+ if literal_type != type_:
+ raise SchemaParseException("Logical type {} requires literal type {}, not {}".format(logical_type, literal_type, type_))
+ return schema_type(other_props)
def make_avsc_object(json_data, names=None):
"""
@@ -892,35 +913,15 @@ def make_avsc_object(json_data, names=None):
@arg names: A Name object (tracks seen names and default space)
"""
- if names == None:
+ if names is None:
names = Names()
# JSON object (non-union)
if callable(getattr(json_data, 'get', None)):
type = json_data.get('type')
other_props = get_other_props(json_data, SCHEMA_RESERVED_PROPS)
- logical_type = None
- if 'logicalType' in json_data:
- logical_type = json_data.get('logicalType')
- if logical_type not in constants.SUPPORTED_LOGICAL_TYPE:
- raise SchemaParseException("Currently does not support %s logical type" % logical_type)
- if type in PRIMITIVE_TYPES:
- if type == 'int' and logical_type == constants.DATE:
- return DateSchema(other_props)
- if type == 'int' and logical_type == constants.TIME_MILLIS:
- return TimeMillisSchema(other_props=other_props)
- if type == 'long' and logical_type == constants.TIME_MICROS:
- return TimeMicrosSchema(other_props=other_props)
- if type == 'long' and logical_type == constants.TIMESTAMP_MILLIS:
- return TimestampMillisSchema(other_props=other_props)
- if type == 'long' and logical_type == constants.TIMESTAMP_MICROS:
- return TimestampMicrosSchema(other_props=other_props)
- if type == 'bytes' and logical_type == constants.DECIMAL:
- precision = json_data.get('precision')
- scale = 0 if json_data.get('scale') is None else json_data.get('scale')
- return BytesDecimalSchema(precision, scale, other_props)
- return PrimitiveSchema(type, other_props)
- elif type in NAMED_TYPES:
+ logical_type = json_data.get('logicalType')
+ if type in NAMED_TYPES:
name = json_data.get('name')
namespace = json_data.get('namespace', names.default_namespace)
if type == 'fixed':
@@ -940,7 +941,11 @@ def make_avsc_object(json_data, names=None):
return RecordSchema(name, namespace, fields, names, type, doc, other_props)
else:
raise SchemaParseException('Unknown Named Type: %s' % type)
- elif type in VALID_TYPES:
+ if logical_type:
+ return make_logical_schema(logical_type, type, other_props or {})
+ if type in PRIMITIVE_TYPES:
+ return PrimitiveSchema(type, other_props)
+ if type in VALID_TYPES:
if type == 'array':
items = json_data.get('items')
return ArraySchema(items, names, other_props)
diff --git a/lang/py/test/test_schema.py b/lang/py/test/test_schema.py
index dd5aa8a..6b2c0ef 100644
--- a/lang/py/test/test_schema.py
+++ b/lang/py/test/test_schema.py
@@ -1,3 +1,6 @@
+#!/usr/bin/env python
+
+##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -5,17 +8,20 @@
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
-#
+#
# https://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-"""
-Test the schema parsing logic.
-"""
+
+"""Test the schema parsing logic."""
+
+from __future__ import absolute_import, division, print_function
+
+import json
import unittest
import set_avro_test_path
@@ -23,356 +29,208 @@ from avro import schema
from avro.schema import AvroException, SchemaParseException
-def print_test_name(test_name):
- print ''
- print test_name
- print '=' * len(test_name)
- print ''
+class TestSchema(object):
+ """A proxy for a schema string that provides useful test metadata."""
-class ExampleSchema(object):
- def __init__(self, schema_string, valid, name='', comment=''):
- self._schema_string = schema_string
- self._valid = valid
- self._name = name or schema_string # default to schema_string for name
+ def __init__(self, data, name='', comment=''):
+ if not isinstance(data, basestring):
+ data = json.dumps(data)
+ self.data = data
+ self.name = name or data # default to data for name
self.comment = comment
- @property
- def schema_string(self):
- return self._schema_string
+ def parse(self):
+ return schema.parse(str(self))
- @property
- def valid(self):
- return self._valid
+ def __str__(self):
+ return str(self.data)
- @property
- def name(self):
- return self._name
-#
-# Example Schemas
-#
+class ValidTestSchema(TestSchema):
+ """A proxy for a valid schema string that provides useful test metadata."""
+ valid = True
+
-def make_primitive_examples():
- examples = []
- for type in schema.PRIMITIVE_TYPES:
- examples.append(ExampleSchema('"%s"' % type, True))
- examples.append(ExampleSchema('{"type": "%s"}' % type, True))
- return examples
+class InvalidTestSchema(ValidTestSchema):
+ """A proxy for an invalid schema string that provides useful test metadata."""
+ valid = False
-PRIMITIVE_EXAMPLES = [
- ExampleSchema('"True"', False),
- ExampleSchema('True', False),
- ExampleSchema('{"no_type": "test"}', False),
- ExampleSchema('{"type": "panther"}', False),
-] + make_primitive_examples()
+
+PRIMITIVE_EXAMPLES = ([
+ InvalidTestSchema('"True"'),
+ InvalidTestSchema('True'),
+ InvalidTestSchema('{"no_type": "test"}'),
+ InvalidTestSchema('{"type": "panther"}'),
+] + [ValidTestSchema('"{}"'.format(t)) for t in schema.PRIMITIVE_TYPES]
+ + [ValidTestSchema({"type": t}) for t in schema.PRIMITIVE_TYPES])
FIXED_EXAMPLES = [
- ExampleSchema('{"type": "fixed", "name": "Test", "size": 1}', True),
- ExampleSchema("""\
- {"type": "fixed",
- "name": "MyFixed",
- "namespace": "org.apache.hadoop.avro",
- "size": 1}
- """, True),
- ExampleSchema("""\
- {"type": "fixed",
- "name": "Missing size"}
- """, False),
- ExampleSchema("""\
- {"type": "fixed",
- "size": 314}
- """, False),
+ ValidTestSchema({"type": "fixed", "name": "Test", "size": 1}),
+ ValidTestSchema({"type": "fixed", "name": "MyFixed", "size": 1,
+ "namespace": "org.apache.hadoop.avro"}),
+ InvalidTestSchema({"type": "fixed", "name": "Missing size"}),
+ InvalidTestSchema({"type": "fixed", "size": 314}),
]
ENUM_EXAMPLES = [
- ExampleSchema('{"type": "enum", "name": "Test", "symbols": ["A", "B"]}', True),
- ExampleSchema("""\
- {"type": "enum",
- "name": "Status",
- "symbols": "Normal Caution Critical"}
- """, False),
- ExampleSchema("""\
- {"type": "enum",
- "name": [ 0, 1, 1, 2, 3, 5, 8 ],
- "symbols": ["Golden", "Mean"]}
- """, False),
- ExampleSchema("""\
- {"type": "enum",
- "symbols" : ["I", "will", "fail", "no", "name"]}
- """, False),
- ExampleSchema("""\
- {"type": "enum",
- "name": "Test"
- "symbols" : ["AA", "AA"]}
- """, False),
+ ValidTestSchema({"type": "enum", "name": "Test", "symbols": ["A", "B"]}),
+ InvalidTestSchema({"type": "enum", "name": "Status", "symbols": "Normal Caution Critical"}),
+ InvalidTestSchema({"type": "enum", "name": [0, 1, 1, 2, 3, 5, 8],
+ "symbols": ["Golden", "Mean"]}),
+ InvalidTestSchema({"type": "enum", "symbols" : ["I", "will", "fail", "no", "name"]}),
+ InvalidTestSchema({"type": "enum", "name": "Test", "symbols": ["AA", "AA"]}),
]
ARRAY_EXAMPLES = [
- ExampleSchema('{"type": "array", "items": "long"}', True),
- ExampleSchema("""\
- {"type": "array",
- "items": {"type": "enum", "name": "Test", "symbols": ["A", "B"]}}
- """, True),
+ ValidTestSchema({"type": "array", "items": "long"}),
+ ValidTestSchema({"type": "array",
+ "items": {"type": "enum", "name": "Test", "symbols": ["A", "B"]}}),
]
MAP_EXAMPLES = [
- ExampleSchema('{"type": "map", "values": "long"}', True),
- ExampleSchema("""\
- {"type": "map",
- "values": {"type": "enum", "name": "Test", "symbols": ["A", "B"]}}
- """, True),
+ ValidTestSchema({"type": "map", "values": "long"}),
+ ValidTestSchema({"type": "map",
+ "values": {"type": "enum", "name": "Test", "symbols": ["A", "B"]}}),
]
UNION_EXAMPLES = [
- ExampleSchema('["string", "null", "long"]', True),
- ExampleSchema('["null", "null"]', False),
- ExampleSchema('["long", "long"]', False),
- ExampleSchema("""\
- [{"type": "array", "items": "long"}
- {"type": "array", "items": "string"}]
- """, False),
+ ValidTestSchema(["string", "null", "long"]),
+ InvalidTestSchema(["null", "null"]),
+ InvalidTestSchema(["long", "long"]),
+ InvalidTestSchema([{"type": "array", "items": "long"},
+ {"type": "array", "items": "string"}]),
]
RECORD_EXAMPLES = [
- ExampleSchema("""\
- {"type": "record",
- "name": "Test",
- "fields": [{"name": "f",
- "type": "long"}]}
- """, True),
- ExampleSchema("""\
- {"type": "error",
- "name": "Test",
- "fields": [{"name": "f",
- "type": "long"}]}
- """, True),
- ExampleSchema("""\
- {"type": "record",
- "name": "Node",
- "fields": [{"name": "label", "type": "string"},
- {"name": "children",
- "type": {"type": "array", "items": "Node"}}]}
- """, True),
- ExampleSchema("""\
- {"type": "record",
- "name": "Lisp",
- "fields": [{"name": "value",
- "type": ["null", "string",
- {"type": "record",
- "name": "Cons",
- "fields": [{"name": "car", "type": "Lisp"},
- {"name": "cdr", "type": "Lisp"}]}]}]}
- """, True),
- ExampleSchema("""\
- {"type": "record",
- "name": "HandshakeRequest",
- "namespace": "org.apache.avro.ipc",
- "fields": [{"name": "clientHash",
- "type": {"type": "fixed", "name": "MD5", "size": 16}},
- {"name": "clientProtocol", "type": ["null", "string"]},
- {"name": "serverHash", "type": "MD5"},
- {"name": "meta",
- "type": ["null", {"type": "map", "values": "bytes"}]}]}
- """, True),
- ExampleSchema("""\
- {"type": "record",
- "name": "HandshakeResponse",
- "namespace": "org.apache.avro.ipc",
- "fields": [{"name": "match",
- "type": {"type": "enum",
- "name": "HandshakeMatch",
- "symbols": ["BOTH", "CLIENT", "NONE"]}},
- {"name": "serverProtocol", "type": ["null", "string"]},
- {"name": "serverHash",
- "type": ["null",
- {"name": "MD5", "size": 16, "type": "fixed"}]},
- {"name": "meta",
- "type": ["null", {"type": "map", "values": "bytes"}]}]}
- """, True),
- ExampleSchema("""\
- {"type": "record",
- "name": "Interop",
- "namespace": "org.apache.avro",
- "fields": [{"name": "intField", "type": "int"},
- {"name": "longField", "type": "long"},
- {"name": "stringField", "type": "string"},
- {"name": "boolField", "type": "boolean"},
- {"name": "floatField", "type": "float"},
- {"name": "doubleField", "type": "double"},
- {"name": "bytesField", "type": "bytes"},
- {"name": "nullField", "type": "null"},
- {"name": "arrayField",
- "type": {"type": "array", "items": "double"}},
- {"name": "mapField",
- "type": {"type": "map",
- "values": {"name": "Foo",
- "type": "record",
- "fields": [{"name": "label",
- "type": "string"}]}}},
- {"name": "unionField",
- "type": ["boolean",
- "double",
- {"type": "array", "items": "bytes"}]},
- {"name": "enumField",
- "type": {"type": "enum",
- "name": "Kind",
- "symbols": ["A", "B", "C"]}},
- {"name": "fixedField",
- "type": {"type": "fixed", "name": "MD5", "size": 16}},
- {"name": "recordField",
- "type": {"type": "record",
- "name": "Node",
- "fields": [{"name": "label", "type": "string"},
- {"name": "children",
- "type": {"type": "array",
- "items": "Node"}}]}}]}
- """, True),
- ExampleSchema("""\
- {"type": "record",
- "name": "ipAddr",
- "fields": [{"name": "addr",
- "type": [{"name": "IPv6", "type": "fixed", "size": 16},
- {"name": "IPv4", "type": "fixed", "size": 4}]}]}
- """, True),
- ExampleSchema("""\
- {"type": "record",
- "name": "Address",
- "fields": [{"type": "string"},
- {"type": "string", "name": "City"}]}
- """, False),
- ExampleSchema("""\
- {"type": "record",
- "name": "Event",
- "fields": [{"name": "Sponsor"},
- {"name": "City", "type": "string"}]}
- """, False),
- ExampleSchema("""\
- {"type": "record",
- "fields": "His vision, from the constantly passing bars,"
- "name", "Rainer"}
- """, False),
- ExampleSchema("""\
- {"name": ["Tom", "Jerry"],
- "type": "record",
- "fields": [{"name": "name", "type": "string"}]}
- """, False),
+ ValidTestSchema({"type": "record", "name": "Test", "fields": [{"name": "f", "type": "long"}]}),
+ ValidTestSchema({"type": "error", "name": "Test", "fields": [{"name": "f", "type": "long"}]}),
+ ValidTestSchema({"type": "record", "name": "Node",
+ "fields": [
+ {"name": "label", "type": "string"},
+ {"name": "children", "type": {"type": "array", "items": "Node"}}]}),
+ ValidTestSchema({"type": "record", "name": "Lisp",
+ "fields": [{"name": "value",
+ "type": ["null", "string",
+ {"type": "record", "name": "Cons",
+ "fields": [{"name": "car", "type": "Lisp"},
+ {"name": "cdr", "type": "Lisp"}]}]}]}),
+ ValidTestSchema({"type": "record", "name": "HandshakeRequest",
+ "namespace": "org.apache.avro.ipc",
+ "fields": [{"name": "clientHash",
+ "type": {"type": "fixed", "name": "MD5", "size": 16}},
+ {"name": "clientProtocol", "type": ["null", "string"]},
+ {"name": "serverHash", "type": "MD5"},
+ {"name": "meta",
+ "type": ["null", {"type": "map", "values": "bytes"}]}]}),
+ ValidTestSchema({"type": "record", "name": "HandshakeResponse",
+ "namespace": "org.apache.avro.ipc",
+ "fields": [{"name": "match",
+ "type": {"type": "enum", "name": "HandshakeMatch",
+ "symbols": ["BOTH", "CLIENT", "NONE"]}},
+ {"name": "serverProtocol", "type": ["null", "string"]},
+ {"name": "serverHash",
+ "type": ["null", {"name": "MD5", "size": 16, "type": "fixed"}]},
+ {"name": "meta",
+ "type": ["null", {"type": "map", "values": "bytes"}]}]}),
+ ValidTestSchema({"type": "record",
+ "name": "Interop",
+ "namespace": "org.apache.avro",
+ "fields": [{"name": "intField", "type": "int"},
+ {"name": "longField", "type": "long"},
+ {"name": "stringField", "type": "string"},
+ {"name": "boolField", "type": "boolean"},
+ {"name": "floatField", "type": "float"},
+ {"name": "doubleField", "type": "double"},
+ {"name": "bytesField", "type": "bytes"},
+ {"name": "nullField", "type": "null"},
+ {"name": "arrayField", "type": {"type": "array", "items": "double"}},
+ {"name": "mapField",
+ "type": {"type": "map",
+ "values": {"name": "Foo",
+ "type": "record",
+ "fields": [{"name": "label", "type": "string"}]}}},
+ {"name": "unionField",
+ "type": ["boolean", "double", {"type": "array", "items": "bytes"}]},
+ {"name": "enumField",
+ "type": {"type": "enum", "name": "Kind", "symbols": ["A", "B", "C"]}},
+ {"name": "fixedField",
+ "type": {"type": "fixed", "name": "MD5", "size": 16}},
+ {"name": "recordField",
+ "type": {"type": "record", "name": "Node",
+ "fields": [{"name": "label", "type": "string"},
+ {"name": "children",
+ "type": {"type": "array",
+ "items": "Node"}}]}}]}),
+ ValidTestSchema({"type": "record", "name": "ipAddr",
+ "fields": [{"name": "addr", "type": [{"name": "IPv6", "type": "fixed", "size": 16},
+ {"name": "IPv4", "type": "fixed", "size": 4}]}]}),
+ InvalidTestSchema({"type": "record", "name": "Address",
+ "fields": [{"type": "string"}, {"type": "string", "name": "City"}]}),
+ InvalidTestSchema({"type": "record", "name": "Event",
+ "fields": [{"name": "Sponsor"}, {"name": "City", "type": "string"}]}),
+ InvalidTestSchema({"type": "record", "name": "Rainer",
+ "fields": "His vision, from the constantly passing bars"}),
+ InvalidTestSchema({"name": ["Tom", "Jerry"], "type": "record",
+ "fields": [{"name": "name", "type": "string"}]}),
]
DOC_EXAMPLES = [
- ExampleSchema("""\
- {"type": "record",
- "name": "TestDoc",
- "doc": "Doc string",
- "fields": [{"name": "name", "type": "string",
- "doc" : "Doc String"}]}
- """, True),
- ExampleSchema("""\
- {"type": "enum", "name": "Test", "symbols": ["A", "B"],
- "doc": "Doc String"}
- """, True),
+ ValidTestSchema({"type": "record", "name": "TestDoc", "doc": "Doc string",
+ "fields": [{"name": "name", "type": "string", "doc" : "Doc String"}]}),
+ ValidTestSchema({"type": "enum", "name": "Test", "symbols": ["A", "B"], "doc": "Doc String"}),
]
OTHER_PROP_EXAMPLES = [
- ExampleSchema("""\
- {"type": "record",
- "name": "TestRecord",
- "cp_string": "string",
- "cp_int": 1,
- "cp_array": [ 1, 2, 3, 4],
- "fields": [ {"name": "f1", "type": "string", "cp_object": {"a":1,"b":2} },
- {"name": "f2", "type": "long", "cp_null": null} ]}
- """, True),
- ExampleSchema("""\
- {"type": "map", "values": "long", "cp_boolean": true}
- """, True),
- ExampleSchema("""\
- {"type": "enum",
- "name": "TestEnum",
- "symbols": [ "one", "two", "three" ],
- "cp_float" : 1.0 }
- """,True),
- ExampleSchema("""\
- {"type": "long",
- "date": "true"}
- """, True)
+ ValidTestSchema({"type": "record", "name": "TestRecord", "cp_string": "string",
+ "cp_int": 1, "cp_array": [1, 2, 3, 4],
+ "fields": [{"name": "f1", "type": "string", "cp_object": {"a": 1,"b": 2}},
+ {"name": "f2", "type": "long", "cp_null": None}]}),
+ ValidTestSchema({"type": "map", "values": "long", "cp_boolean": True}),
+ ValidTestSchema({"type": "enum", "name": "TestEnum",
+ "symbols": ["one", "two", "three"], "cp_float": 1.0}),
]
DECIMAL_LOGICAL_TYPE = [
- ExampleSchema("""{
- "type": "fixed",
- "logicalType": "decimal",
- "name": "TestDecimal",
- "precision": 4,
- "size": 10,
- "scale": 2}""", True),
- ExampleSchema("""{
- "type": "bytes",
- "logicalType": "decimal",
- "precision": 4,
- "scale": 2}""", True)
+ ValidTestSchema({"type": "fixed", "logicalType": "decimal", "name": "TestDecimal", "precision": 4, "size": 10, "scale": 2}),
+ ValidTestSchema({"type": "bytes", "logicalType": "decimal", "precision": 4, "scale": 2}),
+ InvalidTestSchema({"type": "bytes", "logicalType": "decimal", "precision": 2, "scale": -2}),
+ InvalidTestSchema({"type": "bytes", "logicalType": "decimal", "precision": -2, "scale": 2}),
+ InvalidTestSchema({"type": "bytes", "logicalType": "decimal", "precision": 2, "scale": 3}),
+ InvalidTestSchema({"type": "fixed", "logicalType": "decimal", "name": "TestDecimal", "precision": -10, "scale": 2, "size": 5}),
+ InvalidTestSchema({"type": "fixed", "logicalType": "decimal", "name": "TestDecimal", "precision": 2, "scale": 3, "size": 2}),
+ InvalidTestSchema({"type": "fixed", "logicalType": "decimal", "name": "TestDecimal", "precision": 2, "scale": 2, "size": -2}),
]
DATE_LOGICAL_TYPE = [
- ExampleSchema("""{
- "type": "int",
- "logicalType": "date"} """, True),
- ExampleSchema("""{
- "type": "int",
- "logicalType": "date1"} """, False),
- ExampleSchema("""{
- "type": "long",
- "logicalType": "date"} """, False),
+ ValidTestSchema({"type": "int", "logicalType": "date"}),
+ InvalidTestSchema({"type": "int", "logicalType": "date1"}),
+ InvalidTestSchema({"type": "long", "logicalType": "date"}),
]
TIMEMILLIS_LOGICAL_TYPE = [
- ExampleSchema("""{
- "type": "int",
- "logicalType": "time-millis"} """, True),
- ExampleSchema("""{
- "type": "int",
- "logicalType": "time-milis"} """, False),
- ExampleSchema("""{
- "type": "long",
- "logicalType": "time-millis"} """, False),
+ ValidTestSchema({"type": "int", "logicalType": "time-millis"}),
+ InvalidTestSchema({"type": "int", "logicalType": "time-milis"}),
+ InvalidTestSchema({"type": "long", "logicalType": "time-millis"}),
]
TIMEMICROS_LOGICAL_TYPE = [
- ExampleSchema("""{
- "type": "long",
- "logicalType": "time-micros"} """, True),
- ExampleSchema("""{
- "type": "long",
- "logicalType": "time-micro"} """, False),
- ExampleSchema("""{
- "type": "int",
- "logicalType": "time-micros"} """, False),
+ ValidTestSchema({"type": "long", "logicalType": "time-micros"}),
+ InvalidTestSchema({"type": "long", "logicalType": "time-micro"}),
+ InvalidTestSchema({"type": "int", "logicalType": "time-micros"}),
]
TIMESTAMPMILLIS_LOGICAL_TYPE = [
- ExampleSchema("""{
- "type": "long",
- "logicalType": "timestamp-millis"} """, True),
- ExampleSchema("""{
- "type": "long",
- "logicalType": "timestamp-milis"} """, False),
- ExampleSchema("""{
- "type": "int",
- "logicalType": "timestamp-millis"} """, False),
+ ValidTestSchema({"type": "long", "logicalType": "timestamp-millis"}),
+ InvalidTestSchema({"type": "long", "logicalType": "timestamp-milis"}),
+ InvalidTestSchema({"type": "int", "logicalType": "timestamp-millis"}),
]
TIMESTAMPMICROS_LOGICAL_TYPE = [
- ExampleSchema("""{
- "type": "long",
- "logicalType": "timestamp-micros"} """, True),
- ExampleSchema("""{
- "type": "long",
- "logicalType": "timestamp-micro"} """, False),
- ExampleSchema("""{
- "type": "int",
- "logicalType": "timestamp-micros"} """, False),
+ ValidTestSchema({"type": "long", "logicalType": "timestamp-micros"}),
+ InvalidTestSchema({"type": "long", "logicalType": "timestamp-micro"}),
+ InvalidTestSchema({"type": "int", "logicalType": "timestamp-micros"}),
]
-
EXAMPLES = PRIMITIVE_EXAMPLES
EXAMPLES += FIXED_EXAMPLES
EXAMPLES += ENUM_EXAMPLES
@@ -389,81 +247,22 @@ EXAMPLES += TIMESTAMPMILLIS_LOGICAL_TYPE
EXAMPLES += TIMESTAMPMICROS_LOGICAL_TYPE
VALID_EXAMPLES = [e for e in EXAMPLES if e.valid]
+INVALID_EXAMPLES = [e for e in EXAMPLES if not e.valid]
-# TODO(hammer): refactor into harness for examples
-# TODO(hammer): pretty-print detailed output
-# TODO(hammer): make verbose flag
-# TODO(hammer): show strack trace to user
-# TODO(hammer): use logging module?
class TestSchema(unittest.TestCase):
+ """Miscellaneous tests for schema"""
def test_correct_recursive_extraction(self):
+ """A recursive reference within a schema should be the same type every time."""
s = schema.parse('{"type": "record", "name": "X", "fields": [{"name": "y", "type": {"type": "record", "name": "Y", "fields": [{"name": "Z", "type": "X"}]}}]}')
t = schema.parse(str(s.fields[0].type))
# If we've made it this far, the subschema was reasonably stringified; it ccould be reparsed.
self.assertEqual("X", t.fields[0].type.name)
- def test_parse(self):
- correct = 0
- for example in EXAMPLES:
- try:
- schema.parse(example.schema_string)
- if example.valid:
- correct += 1
- else:
- self.fail("Invalid schema was parsed: " + example.schema_string)
- except:
- if not example.valid:
- correct += 1
- else:
- self.fail("Valid schema failed to parse: " + example.schema_string)
-
- fail_msg = "Parse behavior correct on %d out of %d schemas." % \
- (correct, len(EXAMPLES))
- self.assertEqual(correct, len(EXAMPLES), fail_msg)
-
- def test_valid_cast_to_string_after_parse(self):
- """
- Test that the string generated by an Avro Schema object
- is, in fact, a valid Avro schema.
- """
- print_test_name('TEST CAST TO STRING AFTER PARSE')
- correct = 0
- for example in VALID_EXAMPLES:
- schema_data = schema.parse(example.schema_string)
- schema.parse(str(schema_data))
- correct += 1
-
- fail_msg = "Cast to string success on %d out of %d schemas" % \
- (correct, len(VALID_EXAMPLES))
- self.assertEqual(correct, len(VALID_EXAMPLES), fail_msg)
-
- def test_equivalence_after_round_trip(self):
- """
- 1. Given a string, parse it to get Avro schema "original".
- 2. Serialize "original" to a string and parse that string
- to generate Avro schema "round trip".
- 3. Ensure "original" and "round trip" schemas are equivalent.
- """
- print_test_name('TEST ROUND TRIP')
- correct = 0
- for example in VALID_EXAMPLES:
- original_schema = schema.parse(example.schema_string)
- round_trip_schema = schema.parse(str(original_schema))
- if original_schema == round_trip_schema:
- correct += 1
- debug_msg = "%s: ROUND TRIP SUCCESS" % example.name
- else:
- debug_msg = "%s: ROUND TRIP FAILURE" % example.name
- self.fail("Round trip failure: %s, %s, %s" % (example.name, original_schema, str(original_schema)))
-
- fail_msg = "Round trip success on %d out of %d schemas" % \
- (correct, len(VALID_EXAMPLES))
- self.assertEqual(correct, len(VALID_EXAMPLES), fail_msg)
-
# TODO(hammer): more tests
def test_fullname(self):
- """
+ """Test schema full names
+
The fullname is determined in one of the following ways:
* A name and namespace are both specified. For example,
one might use "name": "X", "namespace": "org.foo"
@@ -490,16 +289,15 @@ class TestSchema(unittest.TestCase):
multiple definitions of a fullname if the definitions are
equivalent.
"""
- print_test_name('TEST FULLNAME')
- # name and namespace specified
+ # name and namespace specified
fullname = schema.Name('a', 'o.a.h', None).fullname
self.assertEqual(fullname, 'o.a.h.a')
# fullname and namespace specified
fullname = schema.Name('a.b.c.d', 'o.a.h', None).fullname
self.assertEqual(fullname, 'a.b.c.d')
-
+
# name and default namespace specified
fullname = schema.Name('a', None, 'b.c.d').fullname
self.assertEqual(fullname, 'b.c.d.a')
@@ -516,58 +314,12 @@ class TestSchema(unittest.TestCase):
fullname = schema.Name('a', 'o.a.a', 'o.a.h').fullname
self.assertEqual(fullname, 'o.a.a.a')
- def test_doc_attributes(self):
- print_test_name('TEST DOC ATTRIBUTES')
- correct = 0
- for example in DOC_EXAMPLES:
- original_schema = schema.parse(example.schema_string)
- if original_schema.doc is not None:
- correct += 1
- if original_schema.type == 'record':
- for f in original_schema.fields:
- if f.doc is None:
- self.fail("Failed to preserve 'doc' in fields: " + example.schema_string)
- self.assertEqual(correct,len(DOC_EXAMPLES))
-
- def test_other_attributes(self):
- print_test_name('TEST OTHER ATTRIBUTES')
- correct = 0
- props = {}
- for example in OTHER_PROP_EXAMPLES:
- original_schema = schema.parse(example.schema_string)
- round_trip_schema = schema.parse(str(original_schema))
- self.assertEqual(original_schema.other_props,round_trip_schema.other_props)
- if original_schema.type == "record":
- field_props = 0
- for f in original_schema.fields:
- if f.other_props:
- props.update(f.other_props)
- field_props += 1
- self.assertEqual(field_props,len(original_schema.fields))
- if original_schema.other_props:
- props.update(original_schema.other_props)
- correct += 1
- for k in props:
- v = props[k]
- if k == "cp_boolean":
- self.assertEqual(type(v), bool)
- elif k == "cp_int":
- self.assertEqual(type(v), int)
- elif k == "cp_object":
- self.assertEqual(type(v), dict)
- elif k == "cp_float":
- self.assertEqual(type(v), float)
- elif k == "cp_array":
- self.assertEqual(type(v), list)
- self.assertEqual(correct,len(OTHER_PROP_EXAMPLES))
-
def test_exception_is_not_swallowed_on_parse_error(self):
- print_test_name('TEST EXCEPTION NOT SWALLOWED ON PARSE ERROR')
-
+ """A specific exception message should appear on a json parse error."""
try:
schema.parse('/not/a/real/file')
caught_exception = False
- except schema.SchemaParseException, e:
+ except schema.SchemaParseException as e:
expected_message = 'Error parsing JSON: /not/a/real/file, error = ' \
'No JSON object could be decoded'
self.assertEqual(expected_message, e.args[0])
@@ -575,78 +327,145 @@ class TestSchema(unittest.TestCase):
self.assertTrue(caught_exception, 'Exception was not caught')
- def test_decimal_invalid_schema(self):
- invalid_schemas = [
- ExampleSchema("""{
- "type": "bytes",
- "logicalType": "decimal",
- "precision": 2,
- "scale": -2}""", True),
-
- ExampleSchema("""{
- "type": "bytes",
- "logicalType": "decimal",
- "precision": -2,
- "scale": 2}""", True),
-
- ExampleSchema("""{
- "type": "bytes",
- "logicalType": "decimal",
- "precision": 2,
- "scale": 3}""", True),
-
- ExampleSchema("""{
+ def test_decimal_valid_type(self):
+ fixed_decimal_schema = ValidTestSchema({
"type": "fixed",
"logicalType": "decimal",
"name": "TestDecimal",
- "precision": -10,
+ "precision": 4,
"scale": 2,
- "size": 5}""", True),
-
+ "size": 2})
- ExampleSchema("""{
- "type": "fixed",
+ bytes_decimal_schema = ValidTestSchema({
+ "type": "bytes",
"logicalType": "decimal",
- "name": "TestDecimal",
- "precision": 2,
- "scale": 3,
- "size": 2}""", True)
- ]
-
- for invalid_schema in invalid_schemas:
- self.assertRaises(SchemaParseException, schema.parse, invalid_schema.schema_string)
-
- fixed_invalid_schema_size = ExampleSchema("""{
- "type": "fixed",
- "logicalType": "decimal",
- "name": "TestDecimal",
- "precision": 2,
- "scale": 2,
- "size": -2}""", True)
- self.assertRaises(AvroException, schema.parse, fixed_invalid_schema_size.schema_string)
+ "precision": 4})
- def test_decimal_valid_type(self):
- fixed_decimal_schema = ExampleSchema("""{
- "type": "fixed",
- "logicalType": "decimal",
- "name": "TestDecimal",
- "precision": 4,
- "scale": 2,
- "size": 2}""", True)
-
- bytes_decimal_schema = ExampleSchema("""{
- "type": "bytes",
- "logicalType": "decimal",
- "precision": 4}""", True)
-
- fixed_decimal = schema.parse(fixed_decimal_schema.schema_string)
+ fixed_decimal = fixed_decimal_schema.parse()
self.assertEqual(4, fixed_decimal.get_prop('precision'))
self.assertEqual(2, fixed_decimal.get_prop('scale'))
self.assertEqual(2, fixed_decimal.get_prop('size'))
- bytes_decimal = schema.parse(bytes_decimal_schema.schema_string)
+ bytes_decimal = bytes_decimal_schema.parse()
self.assertEqual(4, bytes_decimal.get_prop('precision'))
self.assertEqual(0, bytes_decimal.get_prop('scale'))
+class SchemaParseTestCase(unittest.TestCase):
+ """Enable generating parse test cases over all the valid and invalid example schema."""
+
+ def __init__(self, test_schema):
+ """Ignore the normal signature for unittest.TestCase because we are generating
+ many test cases from this one class. This is safe as long as the autoloader
+ ignores this class. The autoloader will ignore this class as long as it has
+ no methods starting with `test_`.
+ """
+ super(SchemaParseTestCase, self).__init__(
+ 'parse_valid' if test_schema.valid else 'parse_invalid')
+ self.test_schema = test_schema
+
+ def parse_valid(self):
+ """Parsing a valid schema should not error."""
+ try:
+ self.test_schema.parse()
+ except (schema.AvroException, schema.SchemaParseException):
+ self.fail("Valid schema failed to parse: {!s}".format(self.test_schema))
+
+ def parse_invalid(self):
+ """Parsing an invalid schema should error."""
+ try:
+ self.test_schema.parse()
+ except (schema.AvroException, schema.SchemaParseException):
+ pass
+ else:
+ self.fail("Invalid schema should not have parsed: {!s}".format(self.test_schema))
+
+class RoundTripParseTestCase(unittest.TestCase):
+ """Enable generating round-trip parse test cases over all the valid test schema."""
+
+ def __init__(self, test_schema):
+ """Ignore the normal signature for unittest.TestCase because we are generating
+ many test cases from this one class. This is safe as long as the autoloader
+ ignores this class. The autoloader will ignore this class as long as it has
+ no methods starting with `test_`.
+ """
+ super(RoundTripParseTestCase, self).__init__('parse_round_trip')
+ self.test_schema = test_schema
+
+ def parse_round_trip(self):
+ """The string of a Schema should be parseable to the same Schema."""
+ parsed = self.test_schema.parse()
+ round_trip = schema.parse(str(parsed))
+ self.assertEqual(parsed, round_trip)
+
+class DocAttributesTestCase(unittest.TestCase):
+ """Enable generating document attribute test cases over all the document test schema."""
+
+ def __init__(self, test_schema):
+ """Ignore the normal signature for unittest.TestCase because we are generating
+ many test cases from this one class. This is safe as long as the autoloader
+ ignores this class. The autoloader will ignore this class as long as it has
+ no methods starting with `test_`.
+ """
+ super(DocAttributesTestCase, self).__init__('check_doc_attributes')
+ self.test_schema = test_schema
+
+ def check_doc_attributes(self):
+ """Documentation attributes should be preserved."""
+ sch = self.test_schema.parse()
+ self.assertIsNotNone(sch.doc, "Failed to preserve 'doc' in schema: {!s}".format(self.test_schema))
+ if sch.type == 'record':
+ for f in sch.fields:
+ self.assertIsNotNone(f.doc, "Failed to preserve 'doc' in fields: {!s}".format(self.test_schema))
+
+
+class OtherAttributesTestCase(unittest.TestCase):
+ """Enable generating attribute test cases over all the other-prop test schema."""
+ _type_map = {
+ "cp_array": list,
+ "cp_boolean": bool,
+ "cp_float": float,
+ "cp_int": int,
+ "cp_null": type(None),
+ "cp_object": dict,
+ "cp_string": basestring,
+ }
+
+ def __init__(self, test_schema):
+ """Ignore the normal signature for unittest.TestCase because we are generating
+ many test cases from this one class. This is safe as long as the autoloader
+ ignores this class. The autoloader will ignore this class as long as it has
+ no methods starting with `test_`.
+ """
+ super(OtherAttributesTestCase, self).__init__('check_attributes')
+ self.test_schema = test_schema
+
+ def _check_props(self, props):
+ for k, v in props.items():
+ self.assertIsInstance(v, self._type_map[k])
+
+ def check_attributes(self):
+ """Other attributes and their types on a schema should be preserved."""
+ sch = self.test_schema.parse()
+ round_trip = schema.parse(str(sch))
+ self.assertEqual(sch.other_props, round_trip.other_props,
+ "Properties were not preserved in a round-trip parse.")
+ self._check_props(sch.other_props)
+ if sch.type == "record":
+ field_props = [f.other_props for f in sch.fields if f.other_props]
+ self.assertEqual(len(field_props), len(sch.fields))
+ for p in field_props:
+ self._check_props(p)
+
+
+def load_tests(loader, default_tests, pattern):
+ """Generate test cases across many test schema."""
+ suite = unittest.TestSuite()
+ suite.addTests(loader.loadTestsFromTestCase(TestSchema))
+ suite.addTests(SchemaParseTestCase(ex) for ex in EXAMPLES)
+ suite.addTests(RoundTripParseTestCase(ex) for ex in VALID_EXAMPLES)
+ suite.addTests(DocAttributesTestCase(ex) for ex in DOC_EXAMPLES)
+ suite.addTests(OtherAttributesTestCase(ex) for ex in OTHER_PROP_EXAMPLES)
+ return suite
+
if __name__ == '__main__':
unittest.main()
[avro] 06/23: AVRO-2445: Remove SimpleJSON (#566)
Posted by rs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rskraba pushed a commit to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/avro.git
commit 702770f857a3d554a7806315c867b7478bfdefe7
Author: Michael A. Smith <mi...@syapse.com>
AuthorDate: Thu Jun 27 08:22:51 2019 -0400
AVRO-2445: Remove SimpleJSON (#566)
---
lang/py/lib/simplejson/LICENSE.txt | 19 -
lang/py/lib/simplejson/__init__.py | 318 -----
lang/py/lib/simplejson/_speedups.c | 2329 ------------------------------------
lang/py/lib/simplejson/decoder.py | 354 ------
lang/py/lib/simplejson/encoder.py | 440 -------
lang/py/lib/simplejson/scanner.py | 65 -
lang/py/lib/simplejson/tool.py | 37 -
lang/py/scripts/avro | 8 +-
lang/py/setup.py | 20 +-
lang/py/src/avro/io.py | 7 +-
lang/py/src/avro/protocol.py | 8 +-
lang/py/src/avro/schema.py | 7 +-
lang/py/test/test_script.py | 6 +-
13 files changed, 20 insertions(+), 3598 deletions(-)
diff --git a/lang/py/lib/simplejson/LICENSE.txt b/lang/py/lib/simplejson/LICENSE.txt
deleted file mode 100644
index ad95f29..0000000
--- a/lang/py/lib/simplejson/LICENSE.txt
+++ /dev/null
@@ -1,19 +0,0 @@
-Copyright (c) 2006 Bob Ippolito
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-of the Software, and to permit persons to whom the Software is furnished to do
-so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
diff --git a/lang/py/lib/simplejson/__init__.py b/lang/py/lib/simplejson/__init__.py
deleted file mode 100644
index e98cec9..0000000
--- a/lang/py/lib/simplejson/__init__.py
+++ /dev/null
@@ -1,318 +0,0 @@
-r"""JSON (JavaScript Object Notation) <https://json.org> is a subset of
-JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data
-interchange format.
-
-:mod:`simplejson` exposes an API familiar to users of the standard library
-:mod:`marshal` and :mod:`pickle` modules. It is the externally maintained
-version of the :mod:`json` library contained in Python 2.6, but maintains
-compatibility with Python 2.4 and Python 2.5 and (currently) has
-significant performance advantages, even without using the optional C
-extension for speedups.
-
-Encoding basic Python object hierarchies::
-
- >>> import simplejson as json
- >>> json.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}])
- '["foo", {"bar": ["baz", null, 1.0, 2]}]'
- >>> print json.dumps("\"foo\bar")
- "\"foo\bar"
- >>> print json.dumps(u'\u1234')
- "\u1234"
- >>> print json.dumps('\\')
- "\\"
- >>> print json.dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True)
- {"a": 0, "b": 0, "c": 0}
- >>> from StringIO import StringIO
- >>> io = StringIO()
- >>> json.dump(['streaming API'], io)
- >>> io.getvalue()
- '["streaming API"]'
-
-Compact encoding::
-
- >>> import simplejson as json
- >>> json.dumps([1,2,3,{'4': 5, '6': 7}], separators=(',',':'))
- '[1,2,3,{"4":5,"6":7}]'
-
-Pretty printing::
-
- >>> import simplejson as json
- >>> s = json.dumps({'4': 5, '6': 7}, sort_keys=True, indent=4)
- >>> print '\n'.join([l.rstrip() for l in s.splitlines()])
- {
- "4": 5,
- "6": 7
- }
-
-Decoding JSON::
-
- >>> import simplejson as json
- >>> obj = [u'foo', {u'bar': [u'baz', None, 1.0, 2]}]
- >>> json.loads('["foo", {"bar":["baz", null, 1.0, 2]}]') == obj
- True
- >>> json.loads('"\\"foo\\bar"') == u'"foo\x08ar'
- True
- >>> from StringIO import StringIO
- >>> io = StringIO('["streaming API"]')
- >>> json.load(io)[0] == 'streaming API'
- True
-
-Specializing JSON object decoding::
-
- >>> import simplejson as json
- >>> def as_complex(dct):
- ... if '__complex__' in dct:
- ... return complex(dct['real'], dct['imag'])
- ... return dct
- ...
- >>> json.loads('{"__complex__": true, "real": 1, "imag": 2}',
- ... object_hook=as_complex)
- (1+2j)
- >>> import decimal
- >>> json.loads('1.1', parse_float=decimal.Decimal) == decimal.Decimal('1.1')
- True
-
-Specializing JSON object encoding::
-
- >>> import simplejson as json
- >>> def encode_complex(obj):
- ... if isinstance(obj, complex):
- ... return [obj.real, obj.imag]
- ... raise TypeError(repr(o) + " is not JSON serializable")
- ...
- >>> json.dumps(2 + 1j, default=encode_complex)
- '[2.0, 1.0]'
- >>> json.JSONEncoder(default=encode_complex).encode(2 + 1j)
- '[2.0, 1.0]'
- >>> ''.join(json.JSONEncoder(default=encode_complex).iterencode(2 + 1j))
- '[2.0, 1.0]'
-
-
-Using simplejson.tool from the shell to validate and pretty-print::
-
- $ echo '{"json":"obj"}' | python -m simplejson.tool
- {
- "json": "obj"
- }
- $ echo '{ 1.2:3.4}' | python -m simplejson.tool
- Expecting property name: line 1 column 2 (char 2)
-"""
-__version__ = '2.0.9'
-__all__ = [
- 'dump', 'dumps', 'load', 'loads',
- 'JSONDecoder', 'JSONEncoder',
-]
-
-__author__ = 'Bob Ippolito <bo...@redivi.com>'
-
-from decoder import JSONDecoder
-from encoder import JSONEncoder
-
-_default_encoder = JSONEncoder(
- skipkeys=False,
- ensure_ascii=True,
- check_circular=True,
- allow_nan=True,
- indent=None,
- separators=None,
- encoding='utf-8',
- default=None,
-)
-
-def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True,
- allow_nan=True, cls=None, indent=None, separators=None,
- encoding='utf-8', default=None, **kw):
- """Serialize ``obj`` as a JSON formatted stream to ``fp`` (a
- ``.write()``-supporting file-like object).
-
- If ``skipkeys`` is true then ``dict`` keys that are not basic types
- (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``)
- will be skipped instead of raising a ``TypeError``.
-
- If ``ensure_ascii`` is false, then the some chunks written to ``fp``
- may be ``unicode`` instances, subject to normal Python ``str`` to
- ``unicode`` coercion rules. Unless ``fp.write()`` explicitly
- understands ``unicode`` (as in ``codecs.getwriter()``) this is likely
- to cause an error.
-
- If ``check_circular`` is false, then the circular reference check
- for container types will be skipped and a circular reference will
- result in an ``OverflowError`` (or worse).
-
- If ``allow_nan`` is false, then it will be a ``ValueError`` to
- serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``)
- in strict compliance of the JSON specification, instead of using the
- JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``).
-
- If ``indent`` is a non-negative integer, then JSON array elements and object
- members will be pretty-printed with that indent level. An indent level
- of 0 will only insert newlines. ``None`` is the most compact representation.
-
- If ``separators`` is an ``(item_separator, dict_separator)`` tuple
- then it will be used instead of the default ``(', ', ': ')`` separators.
- ``(',', ':')`` is the most compact JSON representation.
-
- ``encoding`` is the character encoding for str instances, default is UTF-8.
-
- ``default(obj)`` is a function that should return a serializable version
- of obj or raise TypeError. The default simply raises TypeError.
-
- To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the
- ``.default()`` method to serialize additional types), specify it with
- the ``cls`` kwarg.
-
- """
- # cached encoder
- if (not skipkeys and ensure_ascii and
- check_circular and allow_nan and
- cls is None and indent is None and separators is None and
- encoding == 'utf-8' and default is None and not kw):
- iterable = _default_encoder.iterencode(obj)
- else:
- if cls is None:
- cls = JSONEncoder
- iterable = cls(skipkeys=skipkeys, ensure_ascii=ensure_ascii,
- check_circular=check_circular, allow_nan=allow_nan, indent=indent,
- separators=separators, encoding=encoding,
- default=default, **kw).iterencode(obj)
- # could accelerate with writelines in some versions of Python, at
- # a debuggability cost
- for chunk in iterable:
- fp.write(chunk)
-
-
-def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True,
- allow_nan=True, cls=None, indent=None, separators=None,
- encoding='utf-8', default=None, **kw):
- """Serialize ``obj`` to a JSON formatted ``str``.
-
- If ``skipkeys`` is false then ``dict`` keys that are not basic types
- (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``)
- will be skipped instead of raising a ``TypeError``.
-
- If ``ensure_ascii`` is false, then the return value will be a
- ``unicode`` instance subject to normal Python ``str`` to ``unicode``
- coercion rules instead of being escaped to an ASCII ``str``.
-
- If ``check_circular`` is false, then the circular reference check
- for container types will be skipped and a circular reference will
- result in an ``OverflowError`` (or worse).
-
- If ``allow_nan`` is false, then it will be a ``ValueError`` to
- serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in
- strict compliance of the JSON specification, instead of using the
- JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``).
-
- If ``indent`` is a non-negative integer, then JSON array elements and
- object members will be pretty-printed with that indent level. An indent
- level of 0 will only insert newlines. ``None`` is the most compact
- representation.
-
- If ``separators`` is an ``(item_separator, dict_separator)`` tuple
- then it will be used instead of the default ``(', ', ': ')`` separators.
- ``(',', ':')`` is the most compact JSON representation.
-
- ``encoding`` is the character encoding for str instances, default is UTF-8.
-
- ``default(obj)`` is a function that should return a serializable version
- of obj or raise TypeError. The default simply raises TypeError.
-
- To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the
- ``.default()`` method to serialize additional types), specify it with
- the ``cls`` kwarg.
-
- """
- # cached encoder
- if (not skipkeys and ensure_ascii and
- check_circular and allow_nan and
- cls is None and indent is None and separators is None and
- encoding == 'utf-8' and default is None and not kw):
- return _default_encoder.encode(obj)
- if cls is None:
- cls = JSONEncoder
- return cls(
- skipkeys=skipkeys, ensure_ascii=ensure_ascii,
- check_circular=check_circular, allow_nan=allow_nan, indent=indent,
- separators=separators, encoding=encoding, default=default,
- **kw).encode(obj)
-
-
-_default_decoder = JSONDecoder(encoding=None, object_hook=None)
-
-
-def load(fp, encoding=None, cls=None, object_hook=None, parse_float=None,
- parse_int=None, parse_constant=None, **kw):
- """Deserialize ``fp`` (a ``.read()``-supporting file-like object containing
- a JSON document) to a Python object.
-
- If the contents of ``fp`` is encoded with an ASCII based encoding other
- than utf-8 (e.g. latin-1), then an appropriate ``encoding`` name must
- be specified. Encodings that are not ASCII based (such as UCS-2) are
- not allowed, and should be wrapped with
- ``codecs.getreader(fp)(encoding)``, or simply decoded to a ``unicode``
- object and passed to ``loads()``
-
- ``object_hook`` is an optional function that will be called with the
- result of any object literal decode (a ``dict``). The return value of
- ``object_hook`` will be used instead of the ``dict``. This feature
- can be used to implement custom decoders (e.g. JSON-RPC class hinting).
-
- To use a custom ``JSONDecoder`` subclass, specify it with the ``cls``
- kwarg.
-
- """
- return loads(fp.read(),
- encoding=encoding, cls=cls, object_hook=object_hook,
- parse_float=parse_float, parse_int=parse_int,
- parse_constant=parse_constant, **kw)
-
-
-def loads(s, encoding=None, cls=None, object_hook=None, parse_float=None,
- parse_int=None, parse_constant=None, **kw):
- """Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a JSON
- document) to a Python object.
-
- If ``s`` is a ``str`` instance and is encoded with an ASCII based encoding
- other than utf-8 (e.g. latin-1) then an appropriate ``encoding`` name
- must be specified. Encodings that are not ASCII based (such as UCS-2)
- are not allowed and should be decoded to ``unicode`` first.
-
- ``object_hook`` is an optional function that will be called with the
- result of any object literal decode (a ``dict``). The return value of
- ``object_hook`` will be used instead of the ``dict``. This feature
- can be used to implement custom decoders (e.g. JSON-RPC class hinting).
-
- ``parse_float``, if specified, will be called with the string
- of every JSON float to be decoded. By default this is equivalent to
- float(num_str). This can be used to use another datatype or parser
- for JSON floats (e.g. decimal.Decimal).
-
- ``parse_int``, if specified, will be called with the string
- of every JSON int to be decoded. By default this is equivalent to
- int(num_str). This can be used to use another datatype or parser
- for JSON integers (e.g. float).
-
- ``parse_constant``, if specified, will be called with one of the
- following strings: -Infinity, Infinity, NaN, null, true, false.
- This can be used to raise an exception if invalid JSON numbers
- are encountered.
-
- To use a custom ``JSONDecoder`` subclass, specify it with the ``cls``
- kwarg.
-
- """
- if (cls is None and encoding is None and object_hook is None and
- parse_int is None and parse_float is None and
- parse_constant is None and not kw):
- return _default_decoder.decode(s)
- if cls is None:
- cls = JSONDecoder
- if object_hook is not None:
- kw['object_hook'] = object_hook
- if parse_float is not None:
- kw['parse_float'] = parse_float
- if parse_int is not None:
- kw['parse_int'] = parse_int
- if parse_constant is not None:
- kw['parse_constant'] = parse_constant
- return cls(encoding=encoding, **kw).decode(s)
diff --git a/lang/py/lib/simplejson/_speedups.c b/lang/py/lib/simplejson/_speedups.c
deleted file mode 100644
index 23b5f4a..0000000
--- a/lang/py/lib/simplejson/_speedups.c
+++ /dev/null
@@ -1,2329 +0,0 @@
-#include "Python.h"
-#include "structmember.h"
-#if PY_VERSION_HEX < 0x02060000 && !defined(Py_TYPE)
-#define Py_TYPE(ob) (((PyObject*)(ob))->ob_type)
-#endif
-#if PY_VERSION_HEX < 0x02050000 && !defined(PY_SSIZE_T_MIN)
-typedef int Py_ssize_t;
-#define PY_SSIZE_T_MAX INT_MAX
-#define PY_SSIZE_T_MIN INT_MIN
-#define PyInt_FromSsize_t PyInt_FromLong
-#define PyInt_AsSsize_t PyInt_AsLong
-#endif
-#ifndef Py_IS_FINITE
-#define Py_IS_FINITE(X) (!Py_IS_INFINITY(X) && !Py_IS_NAN(X))
-#endif
-
-#ifdef __GNUC__
-#define UNUSED __attribute__((__unused__))
-#else
-#define UNUSED
-#endif
-
-#define DEFAULT_ENCODING "utf-8"
-
-#define PyScanner_Check(op) PyObject_TypeCheck(op, &PyScannerType)
-#define PyScanner_CheckExact(op) (Py_TYPE(op) == &PyScannerType)
-#define PyEncoder_Check(op) PyObject_TypeCheck(op, &PyEncoderType)
-#define PyEncoder_CheckExact(op) (Py_TYPE(op) == &PyEncoderType)
-
-static PyTypeObject PyScannerType;
-static PyTypeObject PyEncoderType;
-
-typedef struct _PyScannerObject {
- PyObject_HEAD
- PyObject *encoding;
- PyObject *strict;
- PyObject *object_hook;
- PyObject *parse_float;
- PyObject *parse_int;
- PyObject *parse_constant;
-} PyScannerObject;
-
-static PyMemberDef scanner_members[] = {
- {"encoding", T_OBJECT, offsetof(PyScannerObject, encoding), READONLY, "encoding"},
- {"strict", T_OBJECT, offsetof(PyScannerObject, strict), READONLY, "strict"},
- {"object_hook", T_OBJECT, offsetof(PyScannerObject, object_hook), READONLY, "object_hook"},
- {"parse_float", T_OBJECT, offsetof(PyScannerObject, parse_float), READONLY, "parse_float"},
- {"parse_int", T_OBJECT, offsetof(PyScannerObject, parse_int), READONLY, "parse_int"},
- {"parse_constant", T_OBJECT, offsetof(PyScannerObject, parse_constant), READONLY, "parse_constant"},
- {NULL}
-};
-
-typedef struct _PyEncoderObject {
- PyObject_HEAD
- PyObject *markers;
- PyObject *defaultfn;
- PyObject *encoder;
- PyObject *indent;
- PyObject *key_separator;
- PyObject *item_separator;
- PyObject *sort_keys;
- PyObject *skipkeys;
- int fast_encode;
- int allow_nan;
-} PyEncoderObject;
-
-static PyMemberDef encoder_members[] = {
- {"markers", T_OBJECT, offsetof(PyEncoderObject, markers), READONLY, "markers"},
- {"default", T_OBJECT, offsetof(PyEncoderObject, defaultfn), READONLY, "default"},
- {"encoder", T_OBJECT, offsetof(PyEncoderObject, encoder), READONLY, "encoder"},
- {"indent", T_OBJECT, offsetof(PyEncoderObject, indent), READONLY, "indent"},
- {"key_separator", T_OBJECT, offsetof(PyEncoderObject, key_separator), READONLY, "key_separator"},
- {"item_separator", T_OBJECT, offsetof(PyEncoderObject, item_separator), READONLY, "item_separator"},
- {"sort_keys", T_OBJECT, offsetof(PyEncoderObject, sort_keys), READONLY, "sort_keys"},
- {"skipkeys", T_OBJECT, offsetof(PyEncoderObject, skipkeys), READONLY, "skipkeys"},
- {NULL}
-};
-
-static Py_ssize_t
-ascii_escape_char(Py_UNICODE c, char *output, Py_ssize_t chars);
-static PyObject *
-ascii_escape_unicode(PyObject *pystr);
-static PyObject *
-ascii_escape_str(PyObject *pystr);
-static PyObject *
-py_encode_basestring_ascii(PyObject* self UNUSED, PyObject *pystr);
-void init_speedups(void);
-static PyObject *
-scan_once_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr);
-static PyObject *
-scan_once_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr);
-static PyObject *
-_build_rval_index_tuple(PyObject *rval, Py_ssize_t idx);
-static PyObject *
-scanner_new(PyTypeObject *type, PyObject *args, PyObject *kwds);
-static int
-scanner_init(PyObject *self, PyObject *args, PyObject *kwds);
-static void
-scanner_dealloc(PyObject *self);
-static int
-scanner_clear(PyObject *self);
-static PyObject *
-encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds);
-static int
-encoder_init(PyObject *self, PyObject *args, PyObject *kwds);
-static void
-encoder_dealloc(PyObject *self);
-static int
-encoder_clear(PyObject *self);
-static int
-encoder_listencode_list(PyEncoderObject *s, PyObject *rval, PyObject *seq, Py_ssize_t indent_level);
-static int
-encoder_listencode_obj(PyEncoderObject *s, PyObject *rval, PyObject *obj, Py_ssize_t indent_level);
-static int
-encoder_listencode_dict(PyEncoderObject *s, PyObject *rval, PyObject *dct, Py_ssize_t indent_level);
-static PyObject *
-_encoded_const(PyObject *const);
-static void
-raise_errmsg(char *msg, PyObject *s, Py_ssize_t end);
-static PyObject *
-encoder_encode_string(PyEncoderObject *s, PyObject *obj);
-static int
-_convertPyInt_AsSsize_t(PyObject *o, Py_ssize_t *size_ptr);
-static PyObject *
-_convertPyInt_FromSsize_t(Py_ssize_t *size_ptr);
-static PyObject *
-encoder_encode_float(PyEncoderObject *s, PyObject *obj);
-
-#define S_CHAR(c) (c >= ' ' && c <= '~' && c != '\\' && c != '"')
-#define IS_WHITESPACE(c) (((c) == ' ') || ((c) == '\t') || ((c) == '\n') || ((c) == '\r'))
-
-#define MIN_EXPANSION 6
-#ifdef Py_UNICODE_WIDE
-#define MAX_EXPANSION (2 * MIN_EXPANSION)
-#else
-#define MAX_EXPANSION MIN_EXPANSION
-#endif
-
-static int
-_convertPyInt_AsSsize_t(PyObject *o, Py_ssize_t *size_ptr)
-{
- /* PyObject to Py_ssize_t converter */
- *size_ptr = PyInt_AsSsize_t(o);
- if (*size_ptr == -1 && PyErr_Occurred());
- return 1;
- return 0;
-}
-
-static PyObject *
-_convertPyInt_FromSsize_t(Py_ssize_t *size_ptr)
-{
- /* Py_ssize_t to PyObject converter */
- return PyInt_FromSsize_t(*size_ptr);
-}
-
-static Py_ssize_t
-ascii_escape_char(Py_UNICODE c, char *output, Py_ssize_t chars)
-{
- /* Escape unicode code point c to ASCII escape sequences
- in char *output. output must have at least 12 bytes unused to
- accommodate an escaped surrogate pair "\uXXXX\uXXXX" */
- output[chars++] = '\\';
- switch (c) {
- case '\\': output[chars++] = (char)c; break;
- case '"': output[chars++] = (char)c; break;
- case '\b': output[chars++] = 'b'; break;
- case '\f': output[chars++] = 'f'; break;
- case '\n': output[chars++] = 'n'; break;
- case '\r': output[chars++] = 'r'; break;
- case '\t': output[chars++] = 't'; break;
- default:
-#ifdef Py_UNICODE_WIDE
- if (c >= 0x10000) {
- /* UTF-16 surrogate pair */
- Py_UNICODE v = c - 0x10000;
- c = 0xd800 | ((v >> 10) & 0x3ff);
- output[chars++] = 'u';
- output[chars++] = "0123456789abcdef"[(c >> 12) & 0xf];
- output[chars++] = "0123456789abcdef"[(c >> 8) & 0xf];
- output[chars++] = "0123456789abcdef"[(c >> 4) & 0xf];
- output[chars++] = "0123456789abcdef"[(c ) & 0xf];
- c = 0xdc00 | (v & 0x3ff);
- output[chars++] = '\\';
- }
-#endif
- output[chars++] = 'u';
- output[chars++] = "0123456789abcdef"[(c >> 12) & 0xf];
- output[chars++] = "0123456789abcdef"[(c >> 8) & 0xf];
- output[chars++] = "0123456789abcdef"[(c >> 4) & 0xf];
- output[chars++] = "0123456789abcdef"[(c ) & 0xf];
- }
- return chars;
-}
-
-static PyObject *
-ascii_escape_unicode(PyObject *pystr)
-{
- /* Take a PyUnicode pystr and return a new ASCII-only escaped PyString */
- Py_ssize_t i;
- Py_ssize_t input_chars;
- Py_ssize_t output_size;
- Py_ssize_t max_output_size;
- Py_ssize_t chars;
- PyObject *rval;
- char *output;
- Py_UNICODE *input_unicode;
-
- input_chars = PyUnicode_GET_SIZE(pystr);
- input_unicode = PyUnicode_AS_UNICODE(pystr);
-
- /* One char input can be up to 6 chars output, estimate 4 of these */
- output_size = 2 + (MIN_EXPANSION * 4) + input_chars;
- max_output_size = 2 + (input_chars * MAX_EXPANSION);
- rval = PyString_FromStringAndSize(NULL, output_size);
- if (rval == NULL) {
- return NULL;
- }
- output = PyString_AS_STRING(rval);
- chars = 0;
- output[chars++] = '"';
- for (i = 0; i < input_chars; i++) {
- Py_UNICODE c = input_unicode[i];
- if (S_CHAR(c)) {
- output[chars++] = (char)c;
- }
- else {
- chars = ascii_escape_char(c, output, chars);
- }
- if (output_size - chars < (1 + MAX_EXPANSION)) {
- /* There's more than four, so let's resize by a lot */
- Py_ssize_t new_output_size = output_size * 2;
- /* This is an upper bound */
- if (new_output_size > max_output_size) {
- new_output_size = max_output_size;
- }
- /* Make sure that the output size changed before resizing */
- if (new_output_size != output_size) {
- output_size = new_output_size;
- if (_PyString_Resize(&rval, output_size) == -1) {
- return NULL;
- }
- output = PyString_AS_STRING(rval);
- }
- }
- }
- output[chars++] = '"';
- if (_PyString_Resize(&rval, chars) == -1) {
- return NULL;
- }
- return rval;
-}
-
-static PyObject *
-ascii_escape_str(PyObject *pystr)
-{
- /* Take a PyString pystr and return a new ASCII-only escaped PyString */
- Py_ssize_t i;
- Py_ssize_t input_chars;
- Py_ssize_t output_size;
- Py_ssize_t chars;
- PyObject *rval;
- char *output;
- char *input_str;
-
- input_chars = PyString_GET_SIZE(pystr);
- input_str = PyString_AS_STRING(pystr);
-
- /* Fast path for a string that's already ASCII */
- for (i = 0; i < input_chars; i++) {
- Py_UNICODE c = (Py_UNICODE)(unsigned char)input_str[i];
- if (!S_CHAR(c)) {
- /* If we have to escape something, scan the string for unicode */
- Py_ssize_t j;
- for (j = i; j < input_chars; j++) {
- c = (Py_UNICODE)(unsigned char)input_str[j];
- if (c > 0x7f) {
- /* We hit a non-ASCII character, bail to unicode mode */
- PyObject *uni;
- uni = PyUnicode_DecodeUTF8(input_str, input_chars, "strict");
- if (uni == NULL) {
- return NULL;
- }
- rval = ascii_escape_unicode(uni);
- Py_DECREF(uni);
- return rval;
- }
- }
- break;
- }
- }
-
- if (i == input_chars) {
- /* Input is already ASCII */
- output_size = 2 + input_chars;
- }
- else {
- /* One char input can be up to 6 chars output, estimate 4 of these */
- output_size = 2 + (MIN_EXPANSION * 4) + input_chars;
- }
- rval = PyString_FromStringAndSize(NULL, output_size);
- if (rval == NULL) {
- return NULL;
- }
- output = PyString_AS_STRING(rval);
- output[0] = '"';
-
- /* We know that everything up to i is ASCII already */
- chars = i + 1;
- memcpy(&output[1], input_str, i);
-
- for (; i < input_chars; i++) {
- Py_UNICODE c = (Py_UNICODE)(unsigned char)input_str[i];
- if (S_CHAR(c)) {
- output[chars++] = (char)c;
- }
- else {
- chars = ascii_escape_char(c, output, chars);
- }
- /* An ASCII char can't possibly expand to a surrogate! */
- if (output_size - chars < (1 + MIN_EXPANSION)) {
- /* There's more than four, so let's resize by a lot */
- output_size *= 2;
- if (output_size > 2 + (input_chars * MIN_EXPANSION)) {
- output_size = 2 + (input_chars * MIN_EXPANSION);
- }
- if (_PyString_Resize(&rval, output_size) == -1) {
- return NULL;
- }
- output = PyString_AS_STRING(rval);
- }
- }
- output[chars++] = '"';
- if (_PyString_Resize(&rval, chars) == -1) {
- return NULL;
- }
- return rval;
-}
-
-static void
-raise_errmsg(char *msg, PyObject *s, Py_ssize_t end)
-{
- /* Use the Python function simplejson.decoder.errmsg to raise a nice
- looking ValueError exception */
- static PyObject *errmsg_fn = NULL;
- PyObject *pymsg;
- if (errmsg_fn == NULL) {
- PyObject *decoder = PyImport_ImportModule("simplejson.decoder");
- if (decoder == NULL)
- return;
- errmsg_fn = PyObject_GetAttrString(decoder, "errmsg");
- Py_DECREF(decoder);
- if (errmsg_fn == NULL)
- return;
- }
- pymsg = PyObject_CallFunction(errmsg_fn, "(zOO&)", msg, s, _convertPyInt_FromSsize_t, &end);
- if (pymsg) {
- PyErr_SetObject(PyExc_ValueError, pymsg);
- Py_DECREF(pymsg);
- }
-}
-
-static PyObject *
-join_list_unicode(PyObject *lst)
-{
- /* return u''.join(lst) */
- static PyObject *joinfn = NULL;
- if (joinfn == NULL) {
- PyObject *ustr = PyUnicode_FromUnicode(NULL, 0);
- if (ustr == NULL)
- return NULL;
-
- joinfn = PyObject_GetAttrString(ustr, "join");
- Py_DECREF(ustr);
- if (joinfn == NULL)
- return NULL;
- }
- return PyObject_CallFunctionObjArgs(joinfn, lst, NULL);
-}
-
-static PyObject *
-join_list_string(PyObject *lst)
-{
- /* return ''.join(lst) */
- static PyObject *joinfn = NULL;
- if (joinfn == NULL) {
- PyObject *ustr = PyString_FromStringAndSize(NULL, 0);
- if (ustr == NULL)
- return NULL;
-
- joinfn = PyObject_GetAttrString(ustr, "join");
- Py_DECREF(ustr);
- if (joinfn == NULL)
- return NULL;
- }
- return PyObject_CallFunctionObjArgs(joinfn, lst, NULL);
-}
-
-static PyObject *
-_build_rval_index_tuple(PyObject *rval, Py_ssize_t idx) {
- /* return (rval, idx) tuple, stealing reference to rval */
- PyObject *tpl;
- PyObject *pyidx;
- /*
- steal a reference to rval, returns (rval, idx)
- */
- if (rval == NULL) {
- return NULL;
- }
- pyidx = PyInt_FromSsize_t(idx);
- if (pyidx == NULL) {
- Py_DECREF(rval);
- return NULL;
- }
- tpl = PyTuple_New(2);
- if (tpl == NULL) {
- Py_DECREF(pyidx);
- Py_DECREF(rval);
- return NULL;
- }
- PyTuple_SET_ITEM(tpl, 0, rval);
- PyTuple_SET_ITEM(tpl, 1, pyidx);
- return tpl;
-}
-
-static PyObject *
-scanstring_str(PyObject *pystr, Py_ssize_t end, char *encoding, int strict, Py_ssize_t *next_end_ptr)
-{
- /* Read the JSON string from PyString pystr.
- end is the index of the first character after the quote.
- encoding is the encoding of pystr (must be an ASCII superset)
- if strict is zero then literal control characters are allowed
- *next_end_ptr is a return-by-reference index of the character
- after the end quote
-
- Return value is a new PyString (if ASCII-only) or PyUnicode
- */
- PyObject *rval;
- Py_ssize_t len = PyString_GET_SIZE(pystr);
- Py_ssize_t begin = end - 1;
- Py_ssize_t next = begin;
- int has_unicode = 0;
- char *buf = PyString_AS_STRING(pystr);
- PyObject *chunks = PyList_New(0);
- if (chunks == NULL) {
- goto bail;
- }
- if (end < 0 || len <= end) {
- PyErr_SetString(PyExc_ValueError, "end is out of bounds");
- goto bail;
- }
- while (1) {
- /* Find the end of the string or the next escape */
- Py_UNICODE c = 0;
- PyObject *chunk = NULL;
- for (next = end; next < len; next++) {
- c = (unsigned char)buf[next];
- if (c == '"' || c == '\\') {
- break;
- }
- else if (strict && c <= 0x1f) {
- raise_errmsg("Invalid control character at", pystr, next);
- goto bail;
- }
- else if (c > 0x7f) {
- has_unicode = 1;
- }
- }
- if (!(c == '"' || c == '\\')) {
- raise_errmsg("Unterminated string starting at", pystr, begin);
- goto bail;
- }
- /* Pick up this chunk if it's not zero length */
- if (next != end) {
- PyObject *strchunk = PyString_FromStringAndSize(&buf[end], next - end);
- if (strchunk == NULL) {
- goto bail;
- }
- if (has_unicode) {
- chunk = PyUnicode_FromEncodedObject(strchunk, encoding, NULL);
- Py_DECREF(strchunk);
- if (chunk == NULL) {
- goto bail;
- }
- }
- else {
- chunk = strchunk;
- }
- if (PyList_Append(chunks, chunk)) {
- Py_DECREF(chunk);
- goto bail;
- }
- Py_DECREF(chunk);
- }
- next++;
- if (c == '"') {
- end = next;
- break;
- }
- if (next == len) {
- raise_errmsg("Unterminated string starting at", pystr, begin);
- goto bail;
- }
- c = buf[next];
- if (c != 'u') {
- /* Non-unicode backslash escapes */
- end = next + 1;
- switch (c) {
- case '"': break;
- case '\\': break;
- case '/': break;
- case 'b': c = '\b'; break;
- case 'f': c = '\f'; break;
- case 'n': c = '\n'; break;
- case 'r': c = '\r'; break;
- case 't': c = '\t'; break;
- default: c = 0;
- }
- if (c == 0) {
- raise_errmsg("Invalid \\escape", pystr, end - 2);
- goto bail;
- }
- }
- else {
- c = 0;
- next++;
- end = next + 4;
- if (end >= len) {
- raise_errmsg("Invalid \\uXXXX escape", pystr, next - 1);
- goto bail;
- }
- /* Decode 4 hex digits */
- for (; next < end; next++) {
- Py_UNICODE digit = buf[next];
- c <<= 4;
- switch (digit) {
- case '0': case '1': case '2': case '3': case '4':
- case '5': case '6': case '7': case '8': case '9':
- c |= (digit - '0'); break;
- case 'a': case 'b': case 'c': case 'd': case 'e':
- case 'f':
- c |= (digit - 'a' + 10); break;
- case 'A': case 'B': case 'C': case 'D': case 'E':
- case 'F':
- c |= (digit - 'A' + 10); break;
- default:
- raise_errmsg("Invalid \\uXXXX escape", pystr, end - 5);
- goto bail;
- }
- }
-#ifdef Py_UNICODE_WIDE
- /* Surrogate pair */
- if ((c & 0xfc00) == 0xd800) {
- Py_UNICODE c2 = 0;
- if (end + 6 >= len) {
- raise_errmsg("Unpaired high surrogate", pystr, end - 5);
- goto bail;
- }
- if (buf[next++] != '\\' || buf[next++] != 'u') {
- raise_errmsg("Unpaired high surrogate", pystr, end - 5);
- goto bail;
- }
- end += 6;
- /* Decode 4 hex digits */
- for (; next < end; next++) {
- c2 <<= 4;
- Py_UNICODE digit = buf[next];
- switch (digit) {
- case '0': case '1': case '2': case '3': case '4':
- case '5': case '6': case '7': case '8': case '9':
- c2 |= (digit - '0'); break;
- case 'a': case 'b': case 'c': case 'd': case 'e':
- case 'f':
- c2 |= (digit - 'a' + 10); break;
- case 'A': case 'B': case 'C': case 'D': case 'E':
- case 'F':
- c2 |= (digit - 'A' + 10); break;
- default:
- raise_errmsg("Invalid \\uXXXX escape", pystr, end - 5);
- goto bail;
- }
- }
- if ((c2 & 0xfc00) != 0xdc00) {
- raise_errmsg("Unpaired high surrogate", pystr, end - 5);
- goto bail;
- }
- c = 0x10000 + (((c - 0xd800) << 10) | (c2 - 0xdc00));
- }
- else if ((c & 0xfc00) == 0xdc00) {
- raise_errmsg("Unpaired low surrogate", pystr, end - 5);
- goto bail;
- }
-#endif
- }
- if (c > 0x7f) {
- has_unicode = 1;
- }
- if (has_unicode) {
- chunk = PyUnicode_FromUnicode(&c, 1);
- if (chunk == NULL) {
- goto bail;
- }
- }
- else {
- char c_char = Py_CHARMASK(c);
- chunk = PyString_FromStringAndSize(&c_char, 1);
- if (chunk == NULL) {
- goto bail;
- }
- }
- if (PyList_Append(chunks, chunk)) {
- Py_DECREF(chunk);
- goto bail;
- }
- Py_DECREF(chunk);
- }
-
- rval = join_list_string(chunks);
- if (rval == NULL) {
- goto bail;
- }
- Py_CLEAR(chunks);
- *next_end_ptr = end;
- return rval;
-bail:
- *next_end_ptr = -1;
- Py_XDECREF(chunks);
- return NULL;
-}
-
-
-static PyObject *
-scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next_end_ptr)
-{
- /* Read the JSON string from PyUnicode pystr.
- end is the index of the first character after the quote.
- if strict is zero then literal control characters are allowed
- *next_end_ptr is a return-by-reference index of the character
- after the end quote
-
- Return value is a new PyUnicode
- */
- PyObject *rval;
- Py_ssize_t len = PyUnicode_GET_SIZE(pystr);
- Py_ssize_t begin = end - 1;
- Py_ssize_t next = begin;
- const Py_UNICODE *buf = PyUnicode_AS_UNICODE(pystr);
- PyObject *chunks = PyList_New(0);
- if (chunks == NULL) {
- goto bail;
- }
- if (end < 0 || len <= end) {
- PyErr_SetString(PyExc_ValueError, "end is out of bounds");
- goto bail;
- }
- while (1) {
- /* Find the end of the string or the next escape */
- Py_UNICODE c = 0;
- PyObject *chunk = NULL;
- for (next = end; next < len; next++) {
- c = buf[next];
- if (c == '"' || c == '\\') {
- break;
- }
- else if (strict && c <= 0x1f) {
- raise_errmsg("Invalid control character at", pystr, next);
- goto bail;
- }
- }
- if (!(c == '"' || c == '\\')) {
- raise_errmsg("Unterminated string starting at", pystr, begin);
- goto bail;
- }
- /* Pick up this chunk if it's not zero length */
- if (next != end) {
- chunk = PyUnicode_FromUnicode(&buf[end], next - end);
- if (chunk == NULL) {
- goto bail;
- }
- if (PyList_Append(chunks, chunk)) {
- Py_DECREF(chunk);
- goto bail;
- }
- Py_DECREF(chunk);
- }
- next++;
- if (c == '"') {
- end = next;
- break;
- }
- if (next == len) {
- raise_errmsg("Unterminated string starting at", pystr, begin);
- goto bail;
- }
- c = buf[next];
- if (c != 'u') {
- /* Non-unicode backslash escapes */
- end = next + 1;
- switch (c) {
- case '"': break;
- case '\\': break;
- case '/': break;
- case 'b': c = '\b'; break;
- case 'f': c = '\f'; break;
- case 'n': c = '\n'; break;
- case 'r': c = '\r'; break;
- case 't': c = '\t'; break;
- default: c = 0;
- }
- if (c == 0) {
- raise_errmsg("Invalid \\escape", pystr, end - 2);
- goto bail;
- }
- }
- else {
- c = 0;
- next++;
- end = next + 4;
- if (end >= len) {
- raise_errmsg("Invalid \\uXXXX escape", pystr, next - 1);
- goto bail;
- }
- /* Decode 4 hex digits */
- for (; next < end; next++) {
- Py_UNICODE digit = buf[next];
- c <<= 4;
- switch (digit) {
- case '0': case '1': case '2': case '3': case '4':
- case '5': case '6': case '7': case '8': case '9':
- c |= (digit - '0'); break;
- case 'a': case 'b': case 'c': case 'd': case 'e':
- case 'f':
- c |= (digit - 'a' + 10); break;
- case 'A': case 'B': case 'C': case 'D': case 'E':
- case 'F':
- c |= (digit - 'A' + 10); break;
- default:
- raise_errmsg("Invalid \\uXXXX escape", pystr, end - 5);
- goto bail;
- }
- }
-#ifdef Py_UNICODE_WIDE
- /* Surrogate pair */
- if ((c & 0xfc00) == 0xd800) {
- Py_UNICODE c2 = 0;
- if (end + 6 >= len) {
- raise_errmsg("Unpaired high surrogate", pystr, end - 5);
- goto bail;
- }
- if (buf[next++] != '\\' || buf[next++] != 'u') {
- raise_errmsg("Unpaired high surrogate", pystr, end - 5);
- goto bail;
- }
- end += 6;
- /* Decode 4 hex digits */
- for (; next < end; next++) {
- c2 <<= 4;
- Py_UNICODE digit = buf[next];
- switch (digit) {
- case '0': case '1': case '2': case '3': case '4':
- case '5': case '6': case '7': case '8': case '9':
- c2 |= (digit - '0'); break;
- case 'a': case 'b': case 'c': case 'd': case 'e':
- case 'f':
- c2 |= (digit - 'a' + 10); break;
- case 'A': case 'B': case 'C': case 'D': case 'E':
- case 'F':
- c2 |= (digit - 'A' + 10); break;
- default:
- raise_errmsg("Invalid \\uXXXX escape", pystr, end - 5);
- goto bail;
- }
- }
- if ((c2 & 0xfc00) != 0xdc00) {
- raise_errmsg("Unpaired high surrogate", pystr, end - 5);
- goto bail;
- }
- c = 0x10000 + (((c - 0xd800) << 10) | (c2 - 0xdc00));
- }
- else if ((c & 0xfc00) == 0xdc00) {
- raise_errmsg("Unpaired low surrogate", pystr, end - 5);
- goto bail;
- }
-#endif
- }
- chunk = PyUnicode_FromUnicode(&c, 1);
- if (chunk == NULL) {
- goto bail;
- }
- if (PyList_Append(chunks, chunk)) {
- Py_DECREF(chunk);
- goto bail;
- }
- Py_DECREF(chunk);
- }
-
- rval = join_list_unicode(chunks);
- if (rval == NULL) {
- goto bail;
- }
- Py_DECREF(chunks);
- *next_end_ptr = end;
- return rval;
-bail:
- *next_end_ptr = -1;
- Py_XDECREF(chunks);
- return NULL;
-}
-
-PyDoc_STRVAR(pydoc_scanstring,
- "scanstring(basestring, end, encoding, strict=True) -> (str, end)\n"
- "\n"
- "Scan the string s for a JSON string. End is the index of the\n"
- "character in s after the quote that started the JSON string.\n"
- "Unescapes all valid JSON string escape sequences and raises ValueError\n"
- "on attempt to decode an invalid string. If strict is False then literal\n"
- "control characters are allowed in the string.\n"
- "\n"
- "Returns a tuple of the decoded string and the index of the character in s\n"
- "after the end quote."
-);
-
-static PyObject *
-py_scanstring(PyObject* self UNUSED, PyObject *args)
-{
- PyObject *pystr;
- PyObject *rval;
- Py_ssize_t end;
- Py_ssize_t next_end = -1;
- char *encoding = NULL;
- int strict = 1;
- if (!PyArg_ParseTuple(args, "OO&|zi:scanstring", &pystr, _convertPyInt_AsSsize_t, &end, &encoding, &strict)) {
- return NULL;
- }
- if (encoding == NULL) {
- encoding = DEFAULT_ENCODING;
- }
- if (PyString_Check(pystr)) {
- rval = scanstring_str(pystr, end, encoding, strict, &next_end);
- }
- else if (PyUnicode_Check(pystr)) {
- rval = scanstring_unicode(pystr, end, strict, &next_end);
- }
- else {
- PyErr_Format(PyExc_TypeError,
- "first argument must be a string, not %.80s",
- Py_TYPE(pystr)->tp_name);
- return NULL;
- }
- return _build_rval_index_tuple(rval, next_end);
-}
-
-PyDoc_STRVAR(pydoc_encode_basestring_ascii,
- "encode_basestring_ascii(basestring) -> str\n"
- "\n"
- "Return an ASCII-only JSON representation of a Python string"
-);
-
-static PyObject *
-py_encode_basestring_ascii(PyObject* self UNUSED, PyObject *pystr)
-{
- /* Return an ASCII-only JSON representation of a Python string */
- /* METH_O */
- if (PyString_Check(pystr)) {
- return ascii_escape_str(pystr);
- }
- else if (PyUnicode_Check(pystr)) {
- return ascii_escape_unicode(pystr);
- }
- else {
- PyErr_Format(PyExc_TypeError,
- "first argument must be a string, not %.80s",
- Py_TYPE(pystr)->tp_name);
- return NULL;
- }
-}
-
-static void
-scanner_dealloc(PyObject *self)
-{
- /* Deallocate scanner object */
- scanner_clear(self);
- Py_TYPE(self)->tp_free(self);
-}
-
-static int
-scanner_traverse(PyObject *self, visitproc visit, void *arg)
-{
- PyScannerObject *s;
- assert(PyScanner_Check(self));
- s = (PyScannerObject *)self;
- Py_VISIT(s->encoding);
- Py_VISIT(s->strict);
- Py_VISIT(s->object_hook);
- Py_VISIT(s->parse_float);
- Py_VISIT(s->parse_int);
- Py_VISIT(s->parse_constant);
- return 0;
-}
-
-static int
-scanner_clear(PyObject *self)
-{
- PyScannerObject *s;
- assert(PyScanner_Check(self));
- s = (PyScannerObject *)self;
- Py_CLEAR(s->encoding);
- Py_CLEAR(s->strict);
- Py_CLEAR(s->object_hook);
- Py_CLEAR(s->parse_float);
- Py_CLEAR(s->parse_int);
- Py_CLEAR(s->parse_constant);
- return 0;
-}
-
-static PyObject *
-_parse_object_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) {
- /* Read a JSON object from PyString pystr.
- idx is the index of the first character after the opening curly brace.
- *next_idx_ptr is a return-by-reference index to the first character after
- the closing curly brace.
-
- Returns a new PyObject (usually a dict, but object_hook can change that)
- */
- char *str = PyString_AS_STRING(pystr);
- Py_ssize_t end_idx = PyString_GET_SIZE(pystr) - 1;
- PyObject *rval = PyDict_New();
- PyObject *key = NULL;
- PyObject *val = NULL;
- char *encoding = PyString_AS_STRING(s->encoding);
- int strict = PyObject_IsTrue(s->strict);
- Py_ssize_t next_idx;
- if (rval == NULL)
- return NULL;
-
- /* skip whitespace after { */
- while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++;
-
- /* only loop if the object is non-empty */
- if (idx <= end_idx && str[idx] != '}') {
- while (idx <= end_idx) {
- /* read key */
- if (str[idx] != '"') {
- raise_errmsg("Expecting property name", pystr, idx);
- goto bail;
- }
- key = scanstring_str(pystr, idx + 1, encoding, strict, &next_idx);
- if (key == NULL)
- goto bail;
- idx = next_idx;
-
- /* skip whitespace between key and : delimiter, read :, skip whitespace */
- while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++;
- if (idx > end_idx || str[idx] != ':') {
- raise_errmsg("Expecting : delimiter", pystr, idx);
- goto bail;
- }
- idx++;
- while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++;
-
- /* read any JSON data type */
- val = scan_once_str(s, pystr, idx, &next_idx);
- if (val == NULL)
- goto bail;
-
- if (PyDict_SetItem(rval, key, val) == -1)
- goto bail;
-
- Py_CLEAR(key);
- Py_CLEAR(val);
- idx = next_idx;
-
- /* skip whitespace before } or , */
- while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++;
-
- /* bail if the object is closed or we didn't get the , delimiter */
- if (idx > end_idx) break;
- if (str[idx] == '}') {
- break;
- }
- else if (str[idx] != ',') {
- raise_errmsg("Expecting , delimiter", pystr, idx);
- goto bail;
- }
- idx++;
-
- /* skip whitespace after , delimiter */
- while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++;
- }
- }
- /* verify that idx < end_idx, str[idx] should be '}' */
- if (idx > end_idx || str[idx] != '}') {
- raise_errmsg("Expecting object", pystr, end_idx);
- goto bail;
- }
- /* if object_hook is not None: rval = object_hook(rval) */
- if (s->object_hook != Py_None) {
- val = PyObject_CallFunctionObjArgs(s->object_hook, rval, NULL);
- if (val == NULL)
- goto bail;
- Py_DECREF(rval);
- rval = val;
- val = NULL;
- }
- *next_idx_ptr = idx + 1;
- return rval;
-bail:
- Py_XDECREF(key);
- Py_XDECREF(val);
- Py_DECREF(rval);
- return NULL;
-}
-
-static PyObject *
-_parse_object_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) {
- /* Read a JSON object from PyUnicode pystr.
- idx is the index of the first character after the opening curly brace.
- *next_idx_ptr is a return-by-reference index to the first character after
- the closing curly brace.
-
- Returns a new PyObject (usually a dict, but object_hook can change that)
- */
- Py_UNICODE *str = PyUnicode_AS_UNICODE(pystr);
- Py_ssize_t end_idx = PyUnicode_GET_SIZE(pystr) - 1;
- PyObject *val = NULL;
- PyObject *rval = PyDict_New();
- PyObject *key = NULL;
- int strict = PyObject_IsTrue(s->strict);
- Py_ssize_t next_idx;
- if (rval == NULL)
- return NULL;
-
- /* skip whitespace after { */
- while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++;
-
- /* only loop if the object is non-empty */
- if (idx <= end_idx && str[idx] != '}') {
- while (idx <= end_idx) {
- /* read key */
- if (str[idx] != '"') {
- raise_errmsg("Expecting property name", pystr, idx);
- goto bail;
- }
- key = scanstring_unicode(pystr, idx + 1, strict, &next_idx);
- if (key == NULL)
- goto bail;
- idx = next_idx;
-
- /* skip whitespace between key and : delimiter, read :, skip whitespace */
- while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++;
- if (idx > end_idx || str[idx] != ':') {
- raise_errmsg("Expecting : delimiter", pystr, idx);
- goto bail;
- }
- idx++;
- while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++;
-
- /* read any JSON term */
- val = scan_once_unicode(s, pystr, idx, &next_idx);
- if (val == NULL)
- goto bail;
-
- if (PyDict_SetItem(rval, key, val) == -1)
- goto bail;
-
- Py_CLEAR(key);
- Py_CLEAR(val);
- idx = next_idx;
-
- /* skip whitespace before } or , */
- while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++;
-
- /* bail if the object is closed or we didn't get the , delimiter */
- if (idx > end_idx) break;
- if (str[idx] == '}') {
- break;
- }
- else if (str[idx] != ',') {
- raise_errmsg("Expecting , delimiter", pystr, idx);
- goto bail;
- }
- idx++;
-
- /* skip whitespace after , delimiter */
- while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++;
- }
- }
-
- /* verify that idx < end_idx, str[idx] should be '}' */
- if (idx > end_idx || str[idx] != '}') {
- raise_errmsg("Expecting object", pystr, end_idx);
- goto bail;
- }
-
- /* if object_hook is not None: rval = object_hook(rval) */
- if (s->object_hook != Py_None) {
- val = PyObject_CallFunctionObjArgs(s->object_hook, rval, NULL);
- if (val == NULL)
- goto bail;
- Py_DECREF(rval);
- rval = val;
- val = NULL;
- }
- *next_idx_ptr = idx + 1;
- return rval;
-bail:
- Py_XDECREF(key);
- Py_XDECREF(val);
- Py_DECREF(rval);
- return NULL;
-}
-
-static PyObject *
-_parse_array_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) {
- /* Read a JSON array from PyString pystr.
- idx is the index of the first character after the opening brace.
- *next_idx_ptr is a return-by-reference index to the first character after
- the closing brace.
-
- Returns a new PyList
- */
- char *str = PyString_AS_STRING(pystr);
- Py_ssize_t end_idx = PyString_GET_SIZE(pystr) - 1;
- PyObject *val = NULL;
- PyObject *rval = PyList_New(0);
- Py_ssize_t next_idx;
- if (rval == NULL)
- return NULL;
-
- /* skip whitespace after [ */
- while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++;
-
- /* only loop if the array is non-empty */
- if (idx <= end_idx && str[idx] != ']') {
- while (idx <= end_idx) {
-
- /* read any JSON term and de-tuplefy the (rval, idx) */
- val = scan_once_str(s, pystr, idx, &next_idx);
- if (val == NULL)
- goto bail;
-
- if (PyList_Append(rval, val) == -1)
- goto bail;
-
- Py_CLEAR(val);
- idx = next_idx;
-
- /* skip whitespace between term and , */
- while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++;
-
- /* bail if the array is closed or we didn't get the , delimiter */
- if (idx > end_idx) break;
- if (str[idx] == ']') {
- break;
- }
- else if (str[idx] != ',') {
- raise_errmsg("Expecting , delimiter", pystr, idx);
- goto bail;
- }
- idx++;
-
- /* skip whitespace after , */
- while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++;
- }
- }
-
- /* verify that idx < end_idx, str[idx] should be ']' */
- if (idx > end_idx || str[idx] != ']') {
- raise_errmsg("Expecting object", pystr, end_idx);
- goto bail;
- }
- *next_idx_ptr = idx + 1;
- return rval;
-bail:
- Py_XDECREF(val);
- Py_DECREF(rval);
- return NULL;
-}
-
-static PyObject *
-_parse_array_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) {
- /* Read a JSON array from PyString pystr.
- idx is the index of the first character after the opening brace.
- *next_idx_ptr is a return-by-reference index to the first character after
- the closing brace.
-
- Returns a new PyList
- */
- Py_UNICODE *str = PyUnicode_AS_UNICODE(pystr);
- Py_ssize_t end_idx = PyUnicode_GET_SIZE(pystr) - 1;
- PyObject *val = NULL;
- PyObject *rval = PyList_New(0);
- Py_ssize_t next_idx;
- if (rval == NULL)
- return NULL;
-
- /* skip whitespace after [ */
- while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++;
-
- /* only loop if the array is non-empty */
- if (idx <= end_idx && str[idx] != ']') {
- while (idx <= end_idx) {
-
- /* read any JSON term */
- val = scan_once_unicode(s, pystr, idx, &next_idx);
- if (val == NULL)
- goto bail;
-
- if (PyList_Append(rval, val) == -1)
- goto bail;
-
- Py_CLEAR(val);
- idx = next_idx;
-
- /* skip whitespace between term and , */
- while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++;
-
- /* bail if the array is closed or we didn't get the , delimiter */
- if (idx > end_idx) break;
- if (str[idx] == ']') {
- break;
- }
- else if (str[idx] != ',') {
- raise_errmsg("Expecting , delimiter", pystr, idx);
- goto bail;
- }
- idx++;
-
- /* skip whitespace after , */
- while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++;
- }
- }
-
- /* verify that idx < end_idx, str[idx] should be ']' */
- if (idx > end_idx || str[idx] != ']') {
- raise_errmsg("Expecting object", pystr, end_idx);
- goto bail;
- }
- *next_idx_ptr = idx + 1;
- return rval;
-bail:
- Py_XDECREF(val);
- Py_DECREF(rval);
- return NULL;
-}
-
-static PyObject *
-_parse_constant(PyScannerObject *s, char *constant, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) {
- /* Read a JSON constant from PyString pystr.
- constant is the constant string that was found
- ("NaN", "Infinity", "-Infinity").
- idx is the index of the first character of the constant
- *next_idx_ptr is a return-by-reference index to the first character after
- the constant.
-
- Returns the result of parse_constant
- */
- PyObject *cstr;
- PyObject *rval;
- /* constant is "NaN", "Infinity", or "-Infinity" */
- cstr = PyString_InternFromString(constant);
- if (cstr == NULL)
- return NULL;
-
- /* rval = parse_constant(constant) */
- rval = PyObject_CallFunctionObjArgs(s->parse_constant, cstr, NULL);
- idx += PyString_GET_SIZE(cstr);
- Py_DECREF(cstr);
- *next_idx_ptr = idx;
- return rval;
-}
-
-static PyObject *
-_match_number_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_ssize_t *next_idx_ptr) {
- /* Read a JSON number from PyString pystr.
- idx is the index of the first character of the number
- *next_idx_ptr is a return-by-reference index to the first character after
- the number.
-
- Returns a new PyObject representation of that number:
- PyInt, PyLong, or PyFloat.
- May return other types if parse_int or parse_float are set
- */
- char *str = PyString_AS_STRING(pystr);
- Py_ssize_t end_idx = PyString_GET_SIZE(pystr) - 1;
- Py_ssize_t idx = start;
- int is_float = 0;
- PyObject *rval;
- PyObject *numstr;
-
- /* read a sign if it's there, make sure it's not the end of the string */
- if (str[idx] == '-') {
- idx++;
- if (idx > end_idx) {
- PyErr_SetNone(PyExc_StopIteration);
- return NULL;
- }
- }
-
- /* read as many integer digits as we find as long as it doesn't start with 0 */
- if (str[idx] >= '1' && str[idx] <= '9') {
- idx++;
- while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++;
- }
- /* if it starts with 0 we only expect one integer digit */
- else if (str[idx] == '0') {
- idx++;
- }
- /* no integer digits, error */
- else {
- PyErr_SetNone(PyExc_StopIteration);
- return NULL;
- }
-
- /* if the next char is '.' followed by a digit then read all float digits */
- if (idx < end_idx && str[idx] == '.' && str[idx + 1] >= '0' && str[idx + 1] <= '9') {
- is_float = 1;
- idx += 2;
- while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++;
- }
-
- /* if the next char is 'e' or 'E' then maybe read the exponent (or backtrack) */
- if (idx < end_idx && (str[idx] == 'e' || str[idx] == 'E')) {
-
- /* save the index of the 'e' or 'E' just in case we need to backtrack */
- Py_ssize_t e_start = idx;
- idx++;
-
- /* read an exponent sign if present */
- if (idx < end_idx && (str[idx] == '-' || str[idx] == '+')) idx++;
-
- /* read all digits */
- while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++;
-
- /* if we got a digit, then parse as float. if not, backtrack */
- if (str[idx - 1] >= '0' && str[idx - 1] <= '9') {
- is_float = 1;
- }
- else {
- idx = e_start;
- }
- }
-
- /* copy the section we determined to be a number */
- numstr = PyString_FromStringAndSize(&str[start], idx - start);
- if (numstr == NULL)
- return NULL;
- if (is_float) {
- /* parse as a float using a fast path if available, otherwise call user defined method */
- if (s->parse_float != (PyObject *)&PyFloat_Type) {
- rval = PyObject_CallFunctionObjArgs(s->parse_float, numstr, NULL);
- }
- else {
- rval = PyFloat_FromDouble(PyOS_ascii_atof(PyString_AS_STRING(numstr)));
- }
- }
- else {
- /* parse as an int using a fast path if available, otherwise call user defined method */
- if (s->parse_int != (PyObject *)&PyInt_Type) {
- rval = PyObject_CallFunctionObjArgs(s->parse_int, numstr, NULL);
- }
- else {
- rval = PyInt_FromString(PyString_AS_STRING(numstr), NULL, 10);
- }
- }
- Py_DECREF(numstr);
- *next_idx_ptr = idx;
- return rval;
-}
-
-static PyObject *
-_match_number_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_ssize_t *next_idx_ptr) {
- /* Read a JSON number from PyUnicode pystr.
- idx is the index of the first character of the number
- *next_idx_ptr is a return-by-reference index to the first character after
- the number.
-
- Returns a new PyObject representation of that number:
- PyInt, PyLong, or PyFloat.
- May return other types if parse_int or parse_float are set
- */
- Py_UNICODE *str = PyUnicode_AS_UNICODE(pystr);
- Py_ssize_t end_idx = PyUnicode_GET_SIZE(pystr) - 1;
- Py_ssize_t idx = start;
- int is_float = 0;
- PyObject *rval;
- PyObject *numstr;
-
- /* read a sign if it's there, make sure it's not the end of the string */
- if (str[idx] == '-') {
- idx++;
- if (idx > end_idx) {
- PyErr_SetNone(PyExc_StopIteration);
- return NULL;
- }
- }
-
- /* read as many integer digits as we find as long as it doesn't start with 0 */
- if (str[idx] >= '1' && str[idx] <= '9') {
- idx++;
- while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++;
- }
- /* if it starts with 0 we only expect one integer digit */
- else if (str[idx] == '0') {
- idx++;
- }
- /* no integer digits, error */
- else {
- PyErr_SetNone(PyExc_StopIteration);
- return NULL;
- }
-
- /* if the next char is '.' followed by a digit then read all float digits */
- if (idx < end_idx && str[idx] == '.' && str[idx + 1] >= '0' && str[idx + 1] <= '9') {
- is_float = 1;
- idx += 2;
- while (idx < end_idx && str[idx] >= '0' && str[idx] <= '9') idx++;
- }
-
- /* if the next char is 'e' or 'E' then maybe read the exponent (or backtrack) */
- if (idx < end_idx && (str[idx] == 'e' || str[idx] == 'E')) {
- Py_ssize_t e_start = idx;
- idx++;
-
- /* read an exponent sign if present */
- if (idx < end_idx && (str[idx] == '-' || str[idx] == '+')) idx++;
-
- /* read all digits */
- while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++;
-
- /* if we got a digit, then parse as float. if not, backtrack */
- if (str[idx - 1] >= '0' && str[idx - 1] <= '9') {
- is_float = 1;
- }
- else {
- idx = e_start;
- }
- }
-
- /* copy the section we determined to be a number */
- numstr = PyUnicode_FromUnicode(&str[start], idx - start);
- if (numstr == NULL)
- return NULL;
- if (is_float) {
- /* parse as a float using a fast path if available, otherwise call user defined method */
- if (s->parse_float != (PyObject *)&PyFloat_Type) {
- rval = PyObject_CallFunctionObjArgs(s->parse_float, numstr, NULL);
- }
- else {
- rval = PyFloat_FromString(numstr, NULL);
- }
- }
- else {
- /* no fast path for unicode -> int, just call */
- rval = PyObject_CallFunctionObjArgs(s->parse_int, numstr, NULL);
- }
- Py_DECREF(numstr);
- *next_idx_ptr = idx;
- return rval;
-}
-
-static PyObject *
-scan_once_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr)
-{
- /* Read one JSON term (of any kind) from PyString pystr.
- idx is the index of the first character of the term
- *next_idx_ptr is a return-by-reference index to the first character after
- the number.
-
- Returns a new PyObject representation of the term.
- */
- char *str = PyString_AS_STRING(pystr);
- Py_ssize_t length = PyString_GET_SIZE(pystr);
- if (idx >= length) {
- PyErr_SetNone(PyExc_StopIteration);
- return NULL;
- }
- switch (str[idx]) {
- case '"':
- /* string */
- return scanstring_str(pystr, idx + 1,
- PyString_AS_STRING(s->encoding),
- PyObject_IsTrue(s->strict),
- next_idx_ptr);
- case '{':
- /* object */
- return _parse_object_str(s, pystr, idx + 1, next_idx_ptr);
- case '[':
- /* array */
- return _parse_array_str(s, pystr, idx + 1, next_idx_ptr);
- case 'n':
- /* null */
- if ((idx + 3 < length) && str[idx + 1] == 'u' && str[idx + 2] == 'l' && str[idx + 3] == 'l') {
- Py_INCREF(Py_None);
- *next_idx_ptr = idx + 4;
- return Py_None;
- }
- break;
- case 't':
- /* true */
- if ((idx + 3 < length) && str[idx + 1] == 'r' && str[idx + 2] == 'u' && str[idx + 3] == 'e') {
- Py_INCREF(Py_True);
- *next_idx_ptr = idx + 4;
- return Py_True;
- }
- break;
- case 'f':
- /* false */
- if ((idx + 4 < length) && str[idx + 1] == 'a' && str[idx + 2] == 'l' && str[idx + 3] == 's' && str[idx + 4] == 'e') {
- Py_INCREF(Py_False);
- *next_idx_ptr = idx + 5;
- return Py_False;
- }
- break;
- case 'N':
- /* NaN */
- if ((idx + 2 < length) && str[idx + 1] == 'a' && str[idx + 2] == 'N') {
- return _parse_constant(s, "NaN", idx, next_idx_ptr);
- }
- break;
- case 'I':
- /* Infinity */
- if ((idx + 7 < length) && str[idx + 1] == 'n' && str[idx + 2] == 'f' && str[idx + 3] == 'i' && str[idx + 4] == 'n' && str[idx + 5] == 'i' && str[idx + 6] == 't' && str[idx + 7] == 'y') {
- return _parse_constant(s, "Infinity", idx, next_idx_ptr);
- }
- break;
- case '-':
- /* -Infinity */
- if ((idx + 8 < length) && str[idx + 1] == 'I' && str[idx + 2] == 'n' && str[idx + 3] == 'f' && str[idx + 4] == 'i' && str[idx + 5] == 'n' && str[idx + 6] == 'i' && str[idx + 7] == 't' && str[idx + 8] == 'y') {
- return _parse_constant(s, "-Infinity", idx, next_idx_ptr);
- }
- break;
- }
- /* Didn't find a string, object, array, or named constant. Look for a number. */
- return _match_number_str(s, pystr, idx, next_idx_ptr);
-}
-
-static PyObject *
-scan_once_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr)
-{
- /* Read one JSON term (of any kind) from PyUnicode pystr.
- idx is the index of the first character of the term
- *next_idx_ptr is a return-by-reference index to the first character after
- the number.
-
- Returns a new PyObject representation of the term.
- */
- Py_UNICODE *str = PyUnicode_AS_UNICODE(pystr);
- Py_ssize_t length = PyUnicode_GET_SIZE(pystr);
- if (idx >= length) {
- PyErr_SetNone(PyExc_StopIteration);
- return NULL;
- }
- switch (str[idx]) {
- case '"':
- /* string */
- return scanstring_unicode(pystr, idx + 1,
- PyObject_IsTrue(s->strict),
- next_idx_ptr);
- case '{':
- /* object */
- return _parse_object_unicode(s, pystr, idx + 1, next_idx_ptr);
- case '[':
- /* array */
- return _parse_array_unicode(s, pystr, idx + 1, next_idx_ptr);
- case 'n':
- /* null */
- if ((idx + 3 < length) && str[idx + 1] == 'u' && str[idx + 2] == 'l' && str[idx + 3] == 'l') {
- Py_INCREF(Py_None);
- *next_idx_ptr = idx + 4;
- return Py_None;
- }
- break;
- case 't':
- /* true */
- if ((idx + 3 < length) && str[idx + 1] == 'r' && str[idx + 2] == 'u' && str[idx + 3] == 'e') {
- Py_INCREF(Py_True);
- *next_idx_ptr = idx + 4;
- return Py_True;
- }
- break;
- case 'f':
- /* false */
- if ((idx + 4 < length) && str[idx + 1] == 'a' && str[idx + 2] == 'l' && str[idx + 3] == 's' && str[idx + 4] == 'e') {
- Py_INCREF(Py_False);
- *next_idx_ptr = idx + 5;
- return Py_False;
- }
- break;
- case 'N':
- /* NaN */
- if ((idx + 2 < length) && str[idx + 1] == 'a' && str[idx + 2] == 'N') {
- return _parse_constant(s, "NaN", idx, next_idx_ptr);
- }
- break;
- case 'I':
- /* Infinity */
- if ((idx + 7 < length) && str[idx + 1] == 'n' && str[idx + 2] == 'f' && str[idx + 3] == 'i' && str[idx + 4] == 'n' && str[idx + 5] == 'i' && str[idx + 6] == 't' && str[idx + 7] == 'y') {
- return _parse_constant(s, "Infinity", idx, next_idx_ptr);
- }
- break;
- case '-':
- /* -Infinity */
- if ((idx + 8 < length) && str[idx + 1] == 'I' && str[idx + 2] == 'n' && str[idx + 3] == 'f' && str[idx + 4] == 'i' && str[idx + 5] == 'n' && str[idx + 6] == 'i' && str[idx + 7] == 't' && str[idx + 8] == 'y') {
- return _parse_constant(s, "-Infinity", idx, next_idx_ptr);
- }
- break;
- }
- /* Didn't find a string, object, array, or named constant. Look for a number. */
- return _match_number_unicode(s, pystr, idx, next_idx_ptr);
-}
-
-static PyObject *
-scanner_call(PyObject *self, PyObject *args, PyObject *kwds)
-{
- /* Python callable interface to scan_once_{str,unicode} */
- PyObject *pystr;
- PyObject *rval;
- Py_ssize_t idx;
- Py_ssize_t next_idx = -1;
- static char *kwlist[] = {"string", "idx", NULL};
- PyScannerObject *s;
- assert(PyScanner_Check(self));
- s = (PyScannerObject *)self;
- if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO&:scan_once", kwlist, &pystr, _convertPyInt_AsSsize_t, &idx))
- return NULL;
-
- if (PyString_Check(pystr)) {
- rval = scan_once_str(s, pystr, idx, &next_idx);
- }
- else if (PyUnicode_Check(pystr)) {
- rval = scan_once_unicode(s, pystr, idx, &next_idx);
- }
- else {
- PyErr_Format(PyExc_TypeError,
- "first argument must be a string, not %.80s",
- Py_TYPE(pystr)->tp_name);
- return NULL;
- }
- return _build_rval_index_tuple(rval, next_idx);
-}
-
-static PyObject *
-scanner_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
-{
- PyScannerObject *s;
- s = (PyScannerObject *)type->tp_alloc(type, 0);
- if (s != NULL) {
- s->encoding = NULL;
- s->strict = NULL;
- s->object_hook = NULL;
- s->parse_float = NULL;
- s->parse_int = NULL;
- s->parse_constant = NULL;
- }
- return (PyObject *)s;
-}
-
-static int
-scanner_init(PyObject *self, PyObject *args, PyObject *kwds)
-{
- /* Initialize Scanner object */
- PyObject *ctx;
- static char *kwlist[] = {"context", NULL};
- PyScannerObject *s;
-
- assert(PyScanner_Check(self));
- s = (PyScannerObject *)self;
-
- if (!PyArg_ParseTupleAndKeywords(args, kwds, "O:make_scanner", kwlist, &ctx))
- return -1;
-
- /* PyString_AS_STRING is used on encoding */
- s->encoding = PyObject_GetAttrString(ctx, "encoding");
- if (s->encoding == Py_None) {
- Py_DECREF(Py_None);
- s->encoding = PyString_InternFromString(DEFAULT_ENCODING);
- }
- else if (PyUnicode_Check(s->encoding)) {
- PyObject *tmp = PyUnicode_AsEncodedString(s->encoding, NULL, NULL);
- Py_DECREF(s->encoding);
- s->encoding = tmp;
- }
- if (s->encoding == NULL || !PyString_Check(s->encoding))
- goto bail;
-
- /* All of these will fail "gracefully" so we don't need to verify them */
- s->strict = PyObject_GetAttrString(ctx, "strict");
- if (s->strict == NULL)
- goto bail;
- s->object_hook = PyObject_GetAttrString(ctx, "object_hook");
- if (s->object_hook == NULL)
- goto bail;
- s->parse_float = PyObject_GetAttrString(ctx, "parse_float");
- if (s->parse_float == NULL)
- goto bail;
- s->parse_int = PyObject_GetAttrString(ctx, "parse_int");
- if (s->parse_int == NULL)
- goto bail;
- s->parse_constant = PyObject_GetAttrString(ctx, "parse_constant");
- if (s->parse_constant == NULL)
- goto bail;
-
- return 0;
-
-bail:
- Py_CLEAR(s->encoding);
- Py_CLEAR(s->strict);
- Py_CLEAR(s->object_hook);
- Py_CLEAR(s->parse_float);
- Py_CLEAR(s->parse_int);
- Py_CLEAR(s->parse_constant);
- return -1;
-}
-
-PyDoc_STRVAR(scanner_doc, "JSON scanner object");
-
-static
-PyTypeObject PyScannerType = {
- PyObject_HEAD_INIT(NULL)
- 0, /* tp_internal */
- "simplejson._speedups.Scanner", /* tp_name */
- sizeof(PyScannerObject), /* tp_basicsize */
- 0, /* tp_itemsize */
- scanner_dealloc, /* tp_dealloc */
- 0, /* tp_print */
- 0, /* tp_getattr */
- 0, /* tp_setattr */
- 0, /* tp_compare */
- 0, /* tp_repr */
- 0, /* tp_as_number */
- 0, /* tp_as_sequence */
- 0, /* tp_as_mapping */
- 0, /* tp_hash */
- scanner_call, /* tp_call */
- 0, /* tp_str */
- 0,/* PyObject_GenericGetAttr, */ /* tp_getattro */
- 0,/* PyObject_GenericSetAttr, */ /* tp_setattro */
- 0, /* tp_as_buffer */
- Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */
- scanner_doc, /* tp_doc */
- scanner_traverse, /* tp_traverse */
- scanner_clear, /* tp_clear */
- 0, /* tp_richcompare */
- 0, /* tp_weaklistoffset */
- 0, /* tp_iter */
- 0, /* tp_iternext */
- 0, /* tp_methods */
- scanner_members, /* tp_members */
- 0, /* tp_getset */
- 0, /* tp_base */
- 0, /* tp_dict */
- 0, /* tp_descr_get */
- 0, /* tp_descr_set */
- 0, /* tp_dictoffset */
- scanner_init, /* tp_init */
- 0,/* PyType_GenericAlloc, */ /* tp_alloc */
- scanner_new, /* tp_new */
- 0,/* PyObject_GC_Del, */ /* tp_free */
-};
-
-static PyObject *
-encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
-{
- PyEncoderObject *s;
- s = (PyEncoderObject *)type->tp_alloc(type, 0);
- if (s != NULL) {
- s->markers = NULL;
- s->defaultfn = NULL;
- s->encoder = NULL;
- s->indent = NULL;
- s->key_separator = NULL;
- s->item_separator = NULL;
- s->sort_keys = NULL;
- s->skipkeys = NULL;
- }
- return (PyObject *)s;
-}
-
-static int
-encoder_init(PyObject *self, PyObject *args, PyObject *kwds)
-{
- /* initialize Encoder object */
- static char *kwlist[] = {"markers", "default", "encoder", "indent", "key_separator", "item_separator", "sort_keys", "skipkeys", "allow_nan", NULL};
-
- PyEncoderObject *s;
- PyObject *allow_nan;
-
- assert(PyEncoder_Check(self));
- s = (PyEncoderObject *)self;
-
- if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOOOOOOOO:make_encoder", kwlist,
- &s->markers, &s->defaultfn, &s->encoder, &s->indent, &s->key_separator, &s->item_separator, &s->sort_keys, &s->skipkeys, &allow_nan))
- return -1;
-
- Py_INCREF(s->markers);
- Py_INCREF(s->defaultfn);
- Py_INCREF(s->encoder);
- Py_INCREF(s->indent);
- Py_INCREF(s->key_separator);
- Py_INCREF(s->item_separator);
- Py_INCREF(s->sort_keys);
- Py_INCREF(s->skipkeys);
- s->fast_encode = (PyCFunction_Check(s->encoder) && PyCFunction_GetFunction(s->encoder) == (PyCFunction)py_encode_basestring_ascii);
- s->allow_nan = PyObject_IsTrue(allow_nan);
- return 0;
-}
-
-static PyObject *
-encoder_call(PyObject *self, PyObject *args, PyObject *kwds)
-{
- /* Python callable interface to encode_listencode_obj */
- static char *kwlist[] = {"obj", "_current_indent_level", NULL};
- PyObject *obj;
- PyObject *rval;
- Py_ssize_t indent_level;
- PyEncoderObject *s;
- assert(PyEncoder_Check(self));
- s = (PyEncoderObject *)self;
- if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO&:_iterencode", kwlist,
- &obj, _convertPyInt_AsSsize_t, &indent_level))
- return NULL;
- rval = PyList_New(0);
- if (rval == NULL)
- return NULL;
- if (encoder_listencode_obj(s, rval, obj, indent_level)) {
- Py_DECREF(rval);
- return NULL;
- }
- return rval;
-}
-
-static PyObject *
-_encoded_const(PyObject *obj)
-{
- /* Return the JSON string representation of None, True, False */
- if (obj == Py_None) {
- static PyObject *s_null = NULL;
- if (s_null == NULL) {
- s_null = PyString_InternFromString("null");
- }
- Py_INCREF(s_null);
- return s_null;
- }
- else if (obj == Py_True) {
- static PyObject *s_true = NULL;
- if (s_true == NULL) {
- s_true = PyString_InternFromString("true");
- }
- Py_INCREF(s_true);
- return s_true;
- }
- else if (obj == Py_False) {
- static PyObject *s_false = NULL;
- if (s_false == NULL) {
- s_false = PyString_InternFromString("false");
- }
- Py_INCREF(s_false);
- return s_false;
- }
- else {
- PyErr_SetString(PyExc_ValueError, "not a const");
- return NULL;
- }
-}
-
-static PyObject *
-encoder_encode_float(PyEncoderObject *s, PyObject *obj)
-{
- /* Return the JSON representation of a PyFloat */
- double i = PyFloat_AS_DOUBLE(obj);
- if (!Py_IS_FINITE(i)) {
- if (!s->allow_nan) {
- PyErr_SetString(PyExc_ValueError, "Out of range float values are not JSON compliant");
- return NULL;
- }
- if (i > 0) {
- return PyString_FromString("Infinity");
- }
- else if (i < 0) {
- return PyString_FromString("-Infinity");
- }
- else {
- return PyString_FromString("NaN");
- }
- }
- /* Use a better float format here? */
- return PyObject_Repr(obj);
-}
-
-static PyObject *
-encoder_encode_string(PyEncoderObject *s, PyObject *obj)
-{
- /* Return the JSON representation of a string */
- if (s->fast_encode)
- return py_encode_basestring_ascii(NULL, obj);
- else
- return PyObject_CallFunctionObjArgs(s->encoder, obj, NULL);
-}
-
-static int
-_steal_list_append(PyObject *lst, PyObject *stolen)
-{
- /* Append stolen and then decrement its reference count */
- int rval = PyList_Append(lst, stolen);
- Py_DECREF(stolen);
- return rval;
-}
-
-static int
-encoder_listencode_obj(PyEncoderObject *s, PyObject *rval, PyObject *obj, Py_ssize_t indent_level)
-{
- /* Encode Python object obj to a JSON term, rval is a PyList */
- PyObject *newobj;
- int rv;
-
- if (obj == Py_None || obj == Py_True || obj == Py_False) {
- PyObject *cstr = _encoded_const(obj);
- if (cstr == NULL)
- return -1;
- return _steal_list_append(rval, cstr);
- }
- else if (PyString_Check(obj) || PyUnicode_Check(obj))
- {
- PyObject *encoded = encoder_encode_string(s, obj);
- if (encoded == NULL)
- return -1;
- return _steal_list_append(rval, encoded);
- }
- else if (PyInt_Check(obj) || PyLong_Check(obj)) {
- PyObject *encoded = PyObject_Str(obj);
- if (encoded == NULL)
- return -1;
- return _steal_list_append(rval, encoded);
- }
- else if (PyFloat_Check(obj)) {
- PyObject *encoded = encoder_encode_float(s, obj);
- if (encoded == NULL)
- return -1;
- return _steal_list_append(rval, encoded);
- }
- else if (PyList_Check(obj) || PyTuple_Check(obj)) {
- return encoder_listencode_list(s, rval, obj, indent_level);
- }
- else if (PyDict_Check(obj)) {
- return encoder_listencode_dict(s, rval, obj, indent_level);
- }
- else {
- PyObject *ident = NULL;
- if (s->markers != Py_None) {
- int has_key;
- ident = PyLong_FromVoidPtr(obj);
- if (ident == NULL)
- return -1;
- has_key = PyDict_Contains(s->markers, ident);
- if (has_key) {
- if (has_key != -1)
- PyErr_SetString(PyExc_ValueError, "Circular reference detected");
- Py_DECREF(ident);
- return -1;
- }
- if (PyDict_SetItem(s->markers, ident, obj)) {
- Py_DECREF(ident);
- return -1;
- }
- }
- newobj = PyObject_CallFunctionObjArgs(s->defaultfn, obj, NULL);
- if (newobj == NULL) {
- Py_XDECREF(ident);
- return -1;
- }
- rv = encoder_listencode_obj(s, rval, newobj, indent_level);
- Py_DECREF(newobj);
- if (rv) {
- Py_XDECREF(ident);
- return -1;
- }
- if (ident != NULL) {
- if (PyDict_DelItem(s->markers, ident)) {
- Py_XDECREF(ident);
- return -1;
- }
- Py_XDECREF(ident);
- }
- return rv;
- }
-}
-
-static int
-encoder_listencode_dict(PyEncoderObject *s, PyObject *rval, PyObject *dct, Py_ssize_t indent_level)
-{
- /* Encode Python dict dct a JSON term, rval is a PyList */
- static PyObject *open_dict = NULL;
- static PyObject *close_dict = NULL;
- static PyObject *empty_dict = NULL;
- PyObject *kstr = NULL;
- PyObject *ident = NULL;
- PyObject *key, *value;
- Py_ssize_t pos;
- int skipkeys;
- Py_ssize_t idx;
-
- if (open_dict == NULL || close_dict == NULL || empty_dict == NULL) {
- open_dict = PyString_InternFromString("{");
- close_dict = PyString_InternFromString("}");
- empty_dict = PyString_InternFromString("{}");
- if (open_dict == NULL || close_dict == NULL || empty_dict == NULL)
- return -1;
- }
- if (PyDict_Size(dct) == 0)
- return PyList_Append(rval, empty_dict);
-
- if (s->markers != Py_None) {
- int has_key;
- ident = PyLong_FromVoidPtr(dct);
- if (ident == NULL)
- goto bail;
- has_key = PyDict_Contains(s->markers, ident);
- if (has_key) {
- if (has_key != -1)
- PyErr_SetString(PyExc_ValueError, "Circular reference detected");
- goto bail;
- }
- if (PyDict_SetItem(s->markers, ident, dct)) {
- goto bail;
- }
- }
-
- if (PyList_Append(rval, open_dict))
- goto bail;
-
- if (s->indent != Py_None) {
- /* TODO: DOES NOT RUN */
- indent_level += 1;
- /*
- newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
- separator = _item_separator + newline_indent
- buf += newline_indent
- */
- }
-
- /* TODO: C speedup not implemented for sort_keys */
-
- pos = 0;
- skipkeys = PyObject_IsTrue(s->skipkeys);
- idx = 0;
- while (PyDict_Next(dct, &pos, &key, &value)) {
- PyObject *encoded;
-
- if (PyString_Check(key) || PyUnicode_Check(key)) {
- Py_INCREF(key);
- kstr = key;
- }
- else if (PyFloat_Check(key)) {
- kstr = encoder_encode_float(s, key);
- if (kstr == NULL)
- goto bail;
- }
- else if (PyInt_Check(key) || PyLong_Check(key)) {
- kstr = PyObject_Str(key);
- if (kstr == NULL)
- goto bail;
- }
- else if (key == Py_True || key == Py_False || key == Py_None) {
- kstr = _encoded_const(key);
- if (kstr == NULL)
- goto bail;
- }
- else if (skipkeys) {
- continue;
- }
- else {
- /* TODO: include repr of key */
- PyErr_SetString(PyExc_ValueError, "keys must be a string");
- goto bail;
- }
-
- if (idx) {
- if (PyList_Append(rval, s->item_separator))
- goto bail;
- }
-
- encoded = encoder_encode_string(s, kstr);
- Py_CLEAR(kstr);
- if (encoded == NULL)
- goto bail;
- if (PyList_Append(rval, encoded)) {
- Py_DECREF(encoded);
- goto bail;
- }
- Py_DECREF(encoded);
- if (PyList_Append(rval, s->key_separator))
- goto bail;
- if (encoder_listencode_obj(s, rval, value, indent_level))
- goto bail;
- idx += 1;
- }
- if (ident != NULL) {
- if (PyDict_DelItem(s->markers, ident))
- goto bail;
- Py_CLEAR(ident);
- }
- if (s->indent != Py_None) {
- /* TODO: DOES NOT RUN */
- indent_level -= 1;
- /*
- yield '\n' + (' ' * (_indent * _current_indent_level))
- */
- }
- if (PyList_Append(rval, close_dict))
- goto bail;
- return 0;
-
-bail:
- Py_XDECREF(kstr);
- Py_XDECREF(ident);
- return -1;
-}
-
-
-static int
-encoder_listencode_list(PyEncoderObject *s, PyObject *rval, PyObject *seq, Py_ssize_t indent_level)
-{
- /* Encode Python list seq to a JSON term, rval is a PyList */
- static PyObject *open_array = NULL;
- static PyObject *close_array = NULL;
- static PyObject *empty_array = NULL;
- PyObject *ident = NULL;
- PyObject *s_fast = NULL;
- Py_ssize_t num_items;
- PyObject **seq_items;
- Py_ssize_t i;
-
- if (open_array == NULL || close_array == NULL || empty_array == NULL) {
- open_array = PyString_InternFromString("[");
- close_array = PyString_InternFromString("]");
- empty_array = PyString_InternFromString("[]");
- if (open_array == NULL || close_array == NULL || empty_array == NULL)
- return -1;
- }
- ident = NULL;
- s_fast = PySequence_Fast(seq, "_iterencode_list needs a sequence");
- if (s_fast == NULL)
- return -1;
- num_items = PySequence_Fast_GET_SIZE(s_fast);
- if (num_items == 0) {
- Py_DECREF(s_fast);
- return PyList_Append(rval, empty_array);
- }
-
- if (s->markers != Py_None) {
- int has_key;
- ident = PyLong_FromVoidPtr(seq);
- if (ident == NULL)
- goto bail;
- has_key = PyDict_Contains(s->markers, ident);
- if (has_key) {
- if (has_key != -1)
- PyErr_SetString(PyExc_ValueError, "Circular reference detected");
- goto bail;
- }
- if (PyDict_SetItem(s->markers, ident, seq)) {
- goto bail;
- }
- }
-
- seq_items = PySequence_Fast_ITEMS(s_fast);
- if (PyList_Append(rval, open_array))
- goto bail;
- if (s->indent != Py_None) {
- /* TODO: DOES NOT RUN */
- indent_level += 1;
- /*
- newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
- separator = _item_separator + newline_indent
- buf += newline_indent
- */
- }
- for (i = 0; i < num_items; i++) {
- PyObject *obj = seq_items[i];
- if (i) {
- if (PyList_Append(rval, s->item_separator))
- goto bail;
- }
- if (encoder_listencode_obj(s, rval, obj, indent_level))
- goto bail;
- }
- if (ident != NULL) {
- if (PyDict_DelItem(s->markers, ident))
- goto bail;
- Py_CLEAR(ident);
- }
- if (s->indent != Py_None) {
- /* TODO: DOES NOT RUN */
- indent_level -= 1;
- /*
- yield '\n' + (' ' * (_indent * _current_indent_level))
- */
- }
- if (PyList_Append(rval, close_array))
- goto bail;
- Py_DECREF(s_fast);
- return 0;
-
-bail:
- Py_XDECREF(ident);
- Py_DECREF(s_fast);
- return -1;
-}
-
-static void
-encoder_dealloc(PyObject *self)
-{
- /* Deallocate Encoder */
- encoder_clear(self);
- Py_TYPE(self)->tp_free(self);
-}
-
-static int
-encoder_traverse(PyObject *self, visitproc visit, void *arg)
-{
- PyEncoderObject *s;
- assert(PyEncoder_Check(self));
- s = (PyEncoderObject *)self;
- Py_VISIT(s->markers);
- Py_VISIT(s->defaultfn);
- Py_VISIT(s->encoder);
- Py_VISIT(s->indent);
- Py_VISIT(s->key_separator);
- Py_VISIT(s->item_separator);
- Py_VISIT(s->sort_keys);
- Py_VISIT(s->skipkeys);
- return 0;
-}
-
-static int
-encoder_clear(PyObject *self)
-{
- /* Deallocate Encoder */
- PyEncoderObject *s;
- assert(PyEncoder_Check(self));
- s = (PyEncoderObject *)self;
- Py_CLEAR(s->markers);
- Py_CLEAR(s->defaultfn);
- Py_CLEAR(s->encoder);
- Py_CLEAR(s->indent);
- Py_CLEAR(s->key_separator);
- Py_CLEAR(s->item_separator);
- Py_CLEAR(s->sort_keys);
- Py_CLEAR(s->skipkeys);
- return 0;
-}
-
-PyDoc_STRVAR(encoder_doc, "_iterencode(obj, _current_indent_level) -> iterable");
-
-static
-PyTypeObject PyEncoderType = {
- PyObject_HEAD_INIT(NULL)
- 0, /* tp_internal */
- "simplejson._speedups.Encoder", /* tp_name */
- sizeof(PyEncoderObject), /* tp_basicsize */
- 0, /* tp_itemsize */
- encoder_dealloc, /* tp_dealloc */
- 0, /* tp_print */
- 0, /* tp_getattr */
- 0, /* tp_setattr */
- 0, /* tp_compare */
- 0, /* tp_repr */
- 0, /* tp_as_number */
- 0, /* tp_as_sequence */
- 0, /* tp_as_mapping */
- 0, /* tp_hash */
- encoder_call, /* tp_call */
- 0, /* tp_str */
- 0, /* tp_getattro */
- 0, /* tp_setattro */
- 0, /* tp_as_buffer */
- Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */
- encoder_doc, /* tp_doc */
- encoder_traverse, /* tp_traverse */
- encoder_clear, /* tp_clear */
- 0, /* tp_richcompare */
- 0, /* tp_weaklistoffset */
- 0, /* tp_iter */
- 0, /* tp_iternext */
- 0, /* tp_methods */
- encoder_members, /* tp_members */
- 0, /* tp_getset */
- 0, /* tp_base */
- 0, /* tp_dict */
- 0, /* tp_descr_get */
- 0, /* tp_descr_set */
- 0, /* tp_dictoffset */
- encoder_init, /* tp_init */
- 0, /* tp_alloc */
- encoder_new, /* tp_new */
- 0, /* tp_free */
-};
-
-static PyMethodDef speedups_methods[] = {
- {"encode_basestring_ascii",
- (PyCFunction)py_encode_basestring_ascii,
- METH_O,
- pydoc_encode_basestring_ascii},
- {"scanstring",
- (PyCFunction)py_scanstring,
- METH_VARARGS,
- pydoc_scanstring},
- {NULL, NULL, 0, NULL}
-};
-
-PyDoc_STRVAR(module_doc,
-"simplejson speedups\n");
-
-void
-init_speedups(void)
-{
- PyObject *m;
- PyScannerType.tp_new = PyType_GenericNew;
- if (PyType_Ready(&PyScannerType) < 0)
- return;
- PyEncoderType.tp_new = PyType_GenericNew;
- if (PyType_Ready(&PyEncoderType) < 0)
- return;
- m = Py_InitModule3("_speedups", speedups_methods, module_doc);
- Py_INCREF((PyObject*)&PyScannerType);
- PyModule_AddObject(m, "make_scanner", (PyObject*)&PyScannerType);
- Py_INCREF((PyObject*)&PyEncoderType);
- PyModule_AddObject(m, "make_encoder", (PyObject*)&PyEncoderType);
-}
diff --git a/lang/py/lib/simplejson/decoder.py b/lang/py/lib/simplejson/decoder.py
deleted file mode 100644
index 0c30e76..0000000
--- a/lang/py/lib/simplejson/decoder.py
+++ /dev/null
@@ -1,354 +0,0 @@
-"""Implementation of JSONDecoder
-"""
-import re
-import sys
-import struct
-
-from simplejson.scanner import make_scanner
-try:
- from simplejson._speedups import scanstring as c_scanstring
-except ImportError:
- c_scanstring = None
-
-__all__ = ['JSONDecoder']
-
-FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL
-
-def _floatconstants():
- _BYTES = '7FF80000000000007FF0000000000000'.decode('hex')
- if sys.byteorder != 'big':
- _BYTES = _BYTES[:8][::-1] + _BYTES[8:][::-1]
- nan, inf = struct.unpack('dd', _BYTES)
- return nan, inf, -inf
-
-NaN, PosInf, NegInf = _floatconstants()
-
-
-def linecol(doc, pos):
- lineno = doc.count('\n', 0, pos) + 1
- if lineno == 1:
- colno = pos
- else:
- colno = pos - doc.rindex('\n', 0, pos)
- return lineno, colno
-
-
-def errmsg(msg, doc, pos, end=None):
- # Note that this function is called from _speedups
- lineno, colno = linecol(doc, pos)
- if end is None:
- #fmt = '{0}: line {1} column {2} (char {3})'
- #return fmt.format(msg, lineno, colno, pos)
- fmt = '%s: line %d column %d (char %d)'
- return fmt % (msg, lineno, colno, pos)
- endlineno, endcolno = linecol(doc, end)
- #fmt = '{0}: line {1} column {2} - line {3} column {4} (char {5} - {6})'
- #return fmt.format(msg, lineno, colno, endlineno, endcolno, pos, end)
- fmt = '%s: line %d column %d - line %d column %d (char %d - %d)'
- return fmt % (msg, lineno, colno, endlineno, endcolno, pos, end)
-
-
-_CONSTANTS = {
- '-Infinity': NegInf,
- 'Infinity': PosInf,
- 'NaN': NaN,
-}
-
-STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS)
-BACKSLASH = {
- '"': u'"', '\\': u'\\', '/': u'/',
- 'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t',
-}
-
-DEFAULT_ENCODING = "utf-8"
-
-def py_scanstring(s, end, encoding=None, strict=True, _b=BACKSLASH, _m=STRINGCHUNK.match):
- """Scan the string s for a JSON string. End is the index of the
- character in s after the quote that started the JSON string.
- Unescapes all valid JSON string escape sequences and raises ValueError
- on attempt to decode an invalid string. If strict is False then literal
- control characters are allowed in the string.
-
- Returns a tuple of the decoded string and the index of the character in s
- after the end quote."""
- if encoding is None:
- encoding = DEFAULT_ENCODING
- chunks = []
- _append = chunks.append
- begin = end - 1
- while 1:
- chunk = _m(s, end)
- if chunk is None:
- raise ValueError(
- errmsg("Unterminated string starting at", s, begin))
- end = chunk.end()
- content, terminator = chunk.groups()
- # Content is contains zero or more unescaped string characters
- if content:
- if not isinstance(content, unicode):
- content = unicode(content, encoding)
- _append(content)
- # Terminator is the end of string, a literal control character,
- # or a backslash denoting that an escape sequence follows
- if terminator == '"':
- break
- elif terminator != '\\':
- if strict:
- msg = "Invalid control character %r at" % (terminator,)
- #msg = "Invalid control character {0!r} at".format(terminator)
- raise ValueError(errmsg(msg, s, end))
- else:
- _append(terminator)
- continue
- try:
- esc = s[end]
- except IndexError:
- raise ValueError(
- errmsg("Unterminated string starting at", s, begin))
- # If not a unicode escape sequence, must be in the lookup table
- if esc != 'u':
- try:
- char = _b[esc]
- except KeyError:
- msg = "Invalid \\escape: " + repr(esc)
- raise ValueError(errmsg(msg, s, end))
- end += 1
- else:
- # Unicode escape sequence
- esc = s[end + 1:end + 5]
- next_end = end + 5
- if len(esc) != 4:
- msg = "Invalid \\uXXXX escape"
- raise ValueError(errmsg(msg, s, end))
- uni = int(esc, 16)
- # Check for surrogate pair on UCS-4 systems
- if 0xd800 <= uni <= 0xdbff and sys.maxunicode > 65535:
- msg = "Invalid \\uXXXX\\uXXXX surrogate pair"
- if not s[end + 5:end + 7] == '\\u':
- raise ValueError(errmsg(msg, s, end))
- esc2 = s[end + 7:end + 11]
- if len(esc2) != 4:
- raise ValueError(errmsg(msg, s, end))
- uni2 = int(esc2, 16)
- uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00))
- next_end += 6
- char = unichr(uni)
- end = next_end
- # Append the unescaped character
- _append(char)
- return u''.join(chunks), end
-
-
-# Use speedup if available
-scanstring = c_scanstring or py_scanstring
-
-WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS)
-WHITESPACE_STR = ' \t\n\r'
-
-def JSONObject((s, end), encoding, strict, scan_once, object_hook, _w=WHITESPACE.match, _ws=WHITESPACE_STR):
- pairs = {}
- # Use a slice to prevent IndexError from being raised, the following
- # check will raise a more specific ValueError if the string is empty
- nextchar = s[end:end + 1]
- # Normally we expect nextchar == '"'
- if nextchar != '"':
- if nextchar in _ws:
- end = _w(s, end).end()
- nextchar = s[end:end + 1]
- # Trivial empty object
- if nextchar == '}':
- return pairs, end + 1
- elif nextchar != '"':
- raise ValueError(errmsg("Expecting property name", s, end))
- end += 1
- while True:
- key, end = scanstring(s, end, encoding, strict)
-
- # To skip some function call overhead we optimize the fast paths where
- # the JSON key separator is ": " or just ":".
- if s[end:end + 1] != ':':
- end = _w(s, end).end()
- if s[end:end + 1] != ':':
- raise ValueError(errmsg("Expecting : delimiter", s, end))
-
- end += 1
-
- try:
- if s[end] in _ws:
- end += 1
- if s[end] in _ws:
- end = _w(s, end + 1).end()
- except IndexError:
- pass
-
- try:
- value, end = scan_once(s, end)
- except StopIteration:
- raise ValueError(errmsg("Expecting object", s, end))
- pairs[key] = value
-
- try:
- nextchar = s[end]
- if nextchar in _ws:
- end = _w(s, end + 1).end()
- nextchar = s[end]
- except IndexError:
- nextchar = ''
- end += 1
-
- if nextchar == '}':
- break
- elif nextchar != ',':
- raise ValueError(errmsg("Expecting , delimiter", s, end - 1))
-
- try:
- nextchar = s[end]
- if nextchar in _ws:
- end += 1
- nextchar = s[end]
- if nextchar in _ws:
- end = _w(s, end + 1).end()
- nextchar = s[end]
- except IndexError:
- nextchar = ''
-
- end += 1
- if nextchar != '"':
- raise ValueError(errmsg("Expecting property name", s, end - 1))
-
- if object_hook is not None:
- pairs = object_hook(pairs)
- return pairs, end
-
-def JSONArray((s, end), scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR):
- values = []
- nextchar = s[end:end + 1]
- if nextchar in _ws:
- end = _w(s, end + 1).end()
- nextchar = s[end:end + 1]
- # Look-ahead for trivial empty array
- if nextchar == ']':
- return values, end + 1
- _append = values.append
- while True:
- try:
- value, end = scan_once(s, end)
- except StopIteration:
- raise ValueError(errmsg("Expecting object", s, end))
- _append(value)
- nextchar = s[end:end + 1]
- if nextchar in _ws:
- end = _w(s, end + 1).end()
- nextchar = s[end:end + 1]
- end += 1
- if nextchar == ']':
- break
- elif nextchar != ',':
- raise ValueError(errmsg("Expecting , delimiter", s, end))
-
- try:
- if s[end] in _ws:
- end += 1
- if s[end] in _ws:
- end = _w(s, end + 1).end()
- except IndexError:
- pass
-
- return values, end
-
-class JSONDecoder(object):
- """Simple JSON <https://json.org> decoder
-
- Performs the following translations in decoding by default:
-
- +---------------+-------------------+
- | JSON | Python |
- +===============+===================+
- | object | dict |
- +---------------+-------------------+
- | array | list |
- +---------------+-------------------+
- | string | unicode |
- +---------------+-------------------+
- | number (int) | int, long |
- +---------------+-------------------+
- | number (real) | float |
- +---------------+-------------------+
- | true | True |
- +---------------+-------------------+
- | false | False |
- +---------------+-------------------+
- | null | None |
- +---------------+-------------------+
-
- It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as
- their corresponding ``float`` values, which is outside the JSON spec.
-
- """
-
- def __init__(self, encoding=None, object_hook=None, parse_float=None,
- parse_int=None, parse_constant=None, strict=True):
- """``encoding`` determines the encoding used to interpret any ``str``
- objects decoded by this instance (utf-8 by default). It has no
- effect when decoding ``unicode`` objects.
-
- Note that currently only encodings that are a superset of ASCII work,
- strings of other encodings should be passed in as ``unicode``.
-
- ``object_hook``, if specified, will be called with the result
- of every JSON object decoded and its return value will be used in
- place of the given ``dict``. This can be used to provide custom
- deserializations (e.g. to support JSON-RPC class hinting).
-
- ``parse_float``, if specified, will be called with the string
- of every JSON float to be decoded. By default this is equivalent to
- float(num_str). This can be used to use another datatype or parser
- for JSON floats (e.g. decimal.Decimal).
-
- ``parse_int``, if specified, will be called with the string
- of every JSON int to be decoded. By default this is equivalent to
- int(num_str). This can be used to use another datatype or parser
- for JSON integers (e.g. float).
-
- ``parse_constant``, if specified, will be called with one of the
- following strings: -Infinity, Infinity, NaN.
- This can be used to raise an exception if invalid JSON numbers
- are encountered.
-
- """
- self.encoding = encoding
- self.object_hook = object_hook
- self.parse_float = parse_float or float
- self.parse_int = parse_int or int
- self.parse_constant = parse_constant or _CONSTANTS.__getitem__
- self.strict = strict
- self.parse_object = JSONObject
- self.parse_array = JSONArray
- self.parse_string = scanstring
- self.scan_once = make_scanner(self)
-
- def decode(self, s, _w=WHITESPACE.match):
- """Return the Python representation of ``s`` (a ``str`` or ``unicode``
- instance containing a JSON document)
-
- """
- obj, end = self.raw_decode(s, idx=_w(s, 0).end())
- end = _w(s, end).end()
- if end != len(s):
- raise ValueError(errmsg("Extra data", s, end, len(s)))
- return obj
-
- def raw_decode(self, s, idx=0):
- """Decode a JSON document from ``s`` (a ``str`` or ``unicode`` beginning
- with a JSON document) and return a 2-tuple of the Python
- representation and the index in ``s`` where the document ended.
-
- This can be used to decode a JSON document from a string that may
- have extraneous data at the end.
-
- """
- try:
- obj, end = self.scan_once(s, idx)
- except StopIteration:
- raise ValueError("No JSON object could be decoded")
- return obj, end
diff --git a/lang/py/lib/simplejson/encoder.py b/lang/py/lib/simplejson/encoder.py
deleted file mode 100644
index 2f3d343..0000000
--- a/lang/py/lib/simplejson/encoder.py
+++ /dev/null
@@ -1,440 +0,0 @@
-"""Implementation of JSONEncoder
-"""
-import re
-
-try:
- from simplejson._speedups import encode_basestring_ascii as c_encode_basestring_ascii
-except ImportError:
- c_encode_basestring_ascii = None
-try:
- from simplejson._speedups import make_encoder as c_make_encoder
-except ImportError:
- c_make_encoder = None
-
-ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]')
-ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])')
-HAS_UTF8 = re.compile(r'[\x80-\xff]')
-ESCAPE_DCT = {
- '\\': '\\\\',
- '"': '\\"',
- '\b': '\\b',
- '\f': '\\f',
- '\n': '\\n',
- '\r': '\\r',
- '\t': '\\t',
-}
-for i in range(0x20):
- #ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i))
- ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
-
-# Assume this produces an infinity on all machines (probably not guaranteed)
-INFINITY = float('1e66666')
-FLOAT_REPR = repr
-
-def encode_basestring(s):
- """Return a JSON representation of a Python string
-
- """
- def replace(match):
- return ESCAPE_DCT[match.group(0)]
- return '"' + ESCAPE.sub(replace, s) + '"'
-
-
-def py_encode_basestring_ascii(s):
- """Return an ASCII-only JSON representation of a Python string
-
- """
- if isinstance(s, str) and HAS_UTF8.search(s) is not None:
- s = s.decode('utf-8')
- def replace(match):
- s = match.group(0)
- try:
- return ESCAPE_DCT[s]
- except KeyError:
- n = ord(s)
- if n < 0x10000:
- #return '\\u{0:04x}'.format(n)
- return '\\u%04x' % (n,)
- else:
- # surrogate pair
- n -= 0x10000
- s1 = 0xd800 | ((n >> 10) & 0x3ff)
- s2 = 0xdc00 | (n & 0x3ff)
- #return '\\u{0:04x}\\u{1:04x}'.format(s1, s2)
- return '\\u%04x\\u%04x' % (s1, s2)
- return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"'
-
-
-encode_basestring_ascii = c_encode_basestring_ascii or py_encode_basestring_ascii
-
-class JSONEncoder(object):
- """Extensible JSON <https://json.org> encoder for Python data structures.
-
- Supports the following objects and types by default:
-
- +-------------------+---------------+
- | Python | JSON |
- +===================+===============+
- | dict | object |
- +-------------------+---------------+
- | list, tuple | array |
- +-------------------+---------------+
- | str, unicode | string |
- +-------------------+---------------+
- | int, long, float | number |
- +-------------------+---------------+
- | True | true |
- +-------------------+---------------+
- | False | false |
- +-------------------+---------------+
- | None | null |
- +-------------------+---------------+
-
- To extend this to recognize other objects, subclass and implement a
- ``.default()`` method with another method that returns a serializable
- object for ``o`` if possible, otherwise it should call the superclass
- implementation (to raise ``TypeError``).
-
- """
- item_separator = ', '
- key_separator = ': '
- def __init__(self, skipkeys=False, ensure_ascii=True,
- check_circular=True, allow_nan=True, sort_keys=False,
- indent=None, separators=None, encoding='utf-8', default=None):
- """Constructor for JSONEncoder, with sensible defaults.
-
- If skipkeys is false, then it is a TypeError to attempt
- encoding of keys that are not str, int, long, float or None. If
- skipkeys is True, such items are simply skipped.
-
- If ensure_ascii is true, the output is guaranteed to be str
- objects with all incoming unicode characters escaped. If
- ensure_ascii is false, the output will be unicode object.
-
- If check_circular is true, then lists, dicts, and custom encoded
- objects will be checked for circular references during encoding to
- prevent an infinite recursion (which would cause an OverflowError).
- Otherwise, no such check takes place.
-
- If allow_nan is true, then NaN, Infinity, and -Infinity will be
- encoded as such. This behavior is not JSON specification compliant,
- but is consistent with most JavaScript based encoders and decoders.
- Otherwise, it will be a ValueError to encode such floats.
-
- If sort_keys is true, then the output of dictionaries will be
- sorted by key; this is useful for regression tests to ensure
- that JSON serializations can be compared on a day-to-day basis.
-
- If indent is a non-negative integer, then JSON array
- elements and object members will be pretty-printed with that
- indent level. An indent level of 0 will only insert newlines.
- None is the most compact representation.
-
- If specified, separators should be a (item_separator, key_separator)
- tuple. The default is (', ', ': '). To get the most compact JSON
- representation you should specify (',', ':') to eliminate whitespace.
-
- If specified, default is a function that gets called for objects
- that can't otherwise be serialized. It should return a JSON encodable
- version of the object or raise a ``TypeError``.
-
- If encoding is not None, then all input strings will be
- transformed into unicode using that encoding prior to JSON-encoding.
- The default is UTF-8.
-
- """
-
- self.skipkeys = skipkeys
- self.ensure_ascii = ensure_ascii
- self.check_circular = check_circular
- self.allow_nan = allow_nan
- self.sort_keys = sort_keys
- self.indent = indent
- if separators is not None:
- self.item_separator, self.key_separator = separators
- if default is not None:
- self.default = default
- self.encoding = encoding
-
- def default(self, o):
- """Implement this method in a subclass such that it returns
- a serializable object for ``o``, or calls the base implementation
- (to raise a ``TypeError``).
-
- For example, to support arbitrary iterators, you could
- implement default like this::
-
- def default(self, o):
- try:
- iterable = iter(o)
- except TypeError:
- pass
- else:
- return list(iterable)
- return JSONEncoder.default(self, o)
-
- """
- raise TypeError(repr(o) + " is not JSON serializable")
-
- def encode(self, o):
- """Return a JSON string representation of a Python data structure.
-
- >>> JSONEncoder().encode({"foo": ["bar", "baz"]})
- '{"foo": ["bar", "baz"]}'
-
- """
- # This is for extremely simple cases and benchmarks.
- if isinstance(o, basestring):
- if isinstance(o, str):
- _encoding = self.encoding
- if (_encoding is not None
- and not (_encoding == 'utf-8')):
- o = o.decode(_encoding)
- if self.ensure_ascii:
- return encode_basestring_ascii(o)
- else:
- return encode_basestring(o)
- # This doesn't pass the iterator directly to ''.join() because the
- # exceptions aren't as detailed. The list call should be roughly
- # equivalent to the PySequence_Fast that ''.join() would do.
- chunks = self.iterencode(o, _one_shot=True)
- if not isinstance(chunks, (list, tuple)):
- chunks = list(chunks)
- return ''.join(chunks)
-
- def iterencode(self, o, _one_shot=False):
- """Encode the given object and yield each string
- representation as available.
-
- For example::
-
- for chunk in JSONEncoder().iterencode(bigobject):
- mysocket.write(chunk)
-
- """
- if self.check_circular:
- markers = {}
- else:
- markers = None
- if self.ensure_ascii:
- _encoder = encode_basestring_ascii
- else:
- _encoder = encode_basestring
- if self.encoding != 'utf-8':
- def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding):
- if isinstance(o, str):
- o = o.decode(_encoding)
- return _orig_encoder(o)
-
- def floatstr(o, allow_nan=self.allow_nan, _repr=FLOAT_REPR, _inf=INFINITY, _neginf=-INFINITY):
- # Check for specials. Note that this type of test is processor- and/or
- # platform-specific, so do tests which don't depend on the internals.
-
- if o != o:
- text = 'NaN'
- elif o == _inf:
- text = 'Infinity'
- elif o == _neginf:
- text = '-Infinity'
- else:
- return _repr(o)
-
- if not allow_nan:
- raise ValueError(
- "Out of range float values are not JSON compliant: " +
- repr(o))
-
- return text
-
-
- if _one_shot and c_make_encoder is not None and not self.indent and not self.sort_keys:
- _iterencode = c_make_encoder(
- markers, self.default, _encoder, self.indent,
- self.key_separator, self.item_separator, self.sort_keys,
- self.skipkeys, self.allow_nan)
- else:
- _iterencode = _make_iterencode(
- markers, self.default, _encoder, self.indent, floatstr,
- self.key_separator, self.item_separator, self.sort_keys,
- self.skipkeys, _one_shot)
- return _iterencode(o, 0)
-
-def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot,
- ## HACK: hand-optimized bytecode; turn globals into locals
- False=False,
- True=True,
- ValueError=ValueError,
- basestring=basestring,
- dict=dict,
- float=float,
- id=id,
- int=int,
- isinstance=isinstance,
- list=list,
- long=long,
- str=str,
- tuple=tuple,
- ):
-
- def _iterencode_list(lst, _current_indent_level):
- if not lst:
- yield '[]'
- return
- if markers is not None:
- markerid = id(lst)
- if markerid in markers:
- raise ValueError("Circular reference detected")
- markers[markerid] = lst
- buf = '['
- if _indent is not None:
- _current_indent_level += 1
- newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
- separator = _item_separator + newline_indent
- buf += newline_indent
- else:
- newline_indent = None
- separator = _item_separator
- first = True
- for value in lst:
- if first:
- first = False
- else:
- buf = separator
- if isinstance(value, basestring):
- yield buf + _encoder(value)
- elif value is None:
- yield buf + 'null'
- elif value is True:
- yield buf + 'true'
- elif value is False:
- yield buf + 'false'
- elif isinstance(value, (int, long)):
- yield buf + str(value)
- elif isinstance(value, float):
- yield buf + _floatstr(value)
- else:
- yield buf
- if isinstance(value, (list, tuple)):
- chunks = _iterencode_list(value, _current_indent_level)
- elif isinstance(value, dict):
- chunks = _iterencode_dict(value, _current_indent_level)
- else:
- chunks = _iterencode(value, _current_indent_level)
- for chunk in chunks:
- yield chunk
- if newline_indent is not None:
- _current_indent_level -= 1
- yield '\n' + (' ' * (_indent * _current_indent_level))
- yield ']'
- if markers is not None:
- del markers[markerid]
-
- def _iterencode_dict(dct, _current_indent_level):
- if not dct:
- yield '{}'
- return
- if markers is not None:
- markerid = id(dct)
- if markerid in markers:
- raise ValueError("Circular reference detected")
- markers[markerid] = dct
- yield '{'
- if _indent is not None:
- _current_indent_level += 1
- newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
- item_separator = _item_separator + newline_indent
- yield newline_indent
- else:
- newline_indent = None
- item_separator = _item_separator
- first = True
- if _sort_keys:
- items = dct.items()
- items.sort(key=lambda kv: kv[0])
- else:
- items = dct.iteritems()
- for key, value in items:
- if isinstance(key, basestring):
- pass
- # JavaScript is weakly typed for these, so it makes sense to
- # also allow them. Many encoders seem to do something like this.
- elif isinstance(key, float):
- key = _floatstr(key)
- elif key is True:
- key = 'true'
- elif key is False:
- key = 'false'
- elif key is None:
- key = 'null'
- elif isinstance(key, (int, long)):
- key = str(key)
- elif _skipkeys:
- continue
- else:
- raise TypeError("key " + repr(key) + " is not a string")
- if first:
- first = False
- else:
- yield item_separator
- yield _encoder(key)
- yield _key_separator
- if isinstance(value, basestring):
- yield _encoder(value)
- elif value is None:
- yield 'null'
- elif value is True:
- yield 'true'
- elif value is False:
- yield 'false'
- elif isinstance(value, (int, long)):
- yield str(value)
- elif isinstance(value, float):
- yield _floatstr(value)
- else:
- if isinstance(value, (list, tuple)):
- chunks = _iterencode_list(value, _current_indent_level)
- elif isinstance(value, dict):
- chunks = _iterencode_dict(value, _current_indent_level)
- else:
- chunks = _iterencode(value, _current_indent_level)
- for chunk in chunks:
- yield chunk
- if newline_indent is not None:
- _current_indent_level -= 1
- yield '\n' + (' ' * (_indent * _current_indent_level))
- yield '}'
- if markers is not None:
- del markers[markerid]
-
- def _iterencode(o, _current_indent_level):
- if isinstance(o, basestring):
- yield _encoder(o)
- elif o is None:
- yield 'null'
- elif o is True:
- yield 'true'
- elif o is False:
- yield 'false'
- elif isinstance(o, (int, long)):
- yield str(o)
- elif isinstance(o, float):
- yield _floatstr(o)
- elif isinstance(o, (list, tuple)):
- for chunk in _iterencode_list(o, _current_indent_level):
- yield chunk
- elif isinstance(o, dict):
- for chunk in _iterencode_dict(o, _current_indent_level):
- yield chunk
- else:
- if markers is not None:
- markerid = id(o)
- if markerid in markers:
- raise ValueError("Circular reference detected")
- markers[markerid] = o
- o = _default(o)
- for chunk in _iterencode(o, _current_indent_level):
- yield chunk
- if markers is not None:
- del markers[markerid]
-
- return _iterencode
diff --git a/lang/py/lib/simplejson/scanner.py b/lang/py/lib/simplejson/scanner.py
deleted file mode 100644
index adbc6ec..0000000
--- a/lang/py/lib/simplejson/scanner.py
+++ /dev/null
@@ -1,65 +0,0 @@
-"""JSON token scanner
-"""
-import re
-try:
- from simplejson._speedups import make_scanner as c_make_scanner
-except ImportError:
- c_make_scanner = None
-
-__all__ = ['make_scanner']
-
-NUMBER_RE = re.compile(
- r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?',
- (re.VERBOSE | re.MULTILINE | re.DOTALL))
-
-def py_make_scanner(context):
- parse_object = context.parse_object
- parse_array = context.parse_array
- parse_string = context.parse_string
- match_number = NUMBER_RE.match
- encoding = context.encoding
- strict = context.strict
- parse_float = context.parse_float
- parse_int = context.parse_int
- parse_constant = context.parse_constant
- object_hook = context.object_hook
-
- def _scan_once(string, idx):
- try:
- nextchar = string[idx]
- except IndexError:
- raise StopIteration
-
- if nextchar == '"':
- return parse_string(string, idx + 1, encoding, strict)
- elif nextchar == '{':
- return parse_object((string, idx + 1), encoding, strict, _scan_once, object_hook)
- elif nextchar == '[':
- return parse_array((string, idx + 1), _scan_once)
- elif nextchar == 'n' and string[idx:idx + 4] == 'null':
- return None, idx + 4
- elif nextchar == 't' and string[idx:idx + 4] == 'true':
- return True, idx + 4
- elif nextchar == 'f' and string[idx:idx + 5] == 'false':
- return False, idx + 5
-
- m = match_number(string, idx)
- if m is not None:
- integer, frac, exp = m.groups()
- if frac or exp:
- res = parse_float(integer + (frac or '') + (exp or ''))
- else:
- res = parse_int(integer)
- return res, m.end()
- elif nextchar == 'N' and string[idx:idx + 3] == 'NaN':
- return parse_constant('NaN'), idx + 3
- elif nextchar == 'I' and string[idx:idx + 8] == 'Infinity':
- return parse_constant('Infinity'), idx + 8
- elif nextchar == '-' and string[idx:idx + 9] == '-Infinity':
- return parse_constant('-Infinity'), idx + 9
- else:
- raise StopIteration
-
- return _scan_once
-
-make_scanner = c_make_scanner or py_make_scanner
diff --git a/lang/py/lib/simplejson/tool.py b/lang/py/lib/simplejson/tool.py
deleted file mode 100644
index 9044331..0000000
--- a/lang/py/lib/simplejson/tool.py
+++ /dev/null
@@ -1,37 +0,0 @@
-r"""Command-line tool to validate and pretty-print JSON
-
-Usage::
-
- $ echo '{"json":"obj"}' | python -m simplejson.tool
- {
- "json": "obj"
- }
- $ echo '{ 1.2:3.4}' | python -m simplejson.tool
- Expecting property name: line 1 column 2 (char 2)
-
-"""
-import sys
-import simplejson
-
-def main():
- if len(sys.argv) == 1:
- infile = sys.stdin
- outfile = sys.stdout
- elif len(sys.argv) == 2:
- infile = open(sys.argv[1], 'rb')
- outfile = sys.stdout
- elif len(sys.argv) == 3:
- infile = open(sys.argv[1], 'rb')
- outfile = open(sys.argv[2], 'wb')
- else:
- raise SystemExit(sys.argv[0] + " [infile [outfile]]")
- try:
- obj = simplejson.load(infile)
- except ValueError, e:
- raise SystemExit(e)
- simplejson.dump(obj, outfile, sort_keys=True, indent=4)
- outfile.write('\n')
-
-
-if __name__ == '__main__':
- main()
diff --git a/lang/py/scripts/avro b/lang/py/scripts/avro
index bc8217f..b320a39 100644
--- a/lang/py/scripts/avro
+++ b/lang/py/scripts/avro
@@ -15,16 +15,14 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-"""Command line utlity for reading and writing Avro files."""
+
+"""Command line utility for reading and writing Avro files."""
from avro.io import DatumReader, DatumWriter
from avro.datafile import DataFileReader, DataFileWriter
import avro.schema
-try:
- import json
-except ImportError:
- import simplejson as json
+import json
import csv
from sys import stdout, stdin
from itertools import ifilter, imap
diff --git a/lang/py/setup.py b/lang/py/setup.py
index 9e49d4c..6902116 100755
--- a/lang/py/setup.py
+++ b/lang/py/setup.py
@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#!/usr/bin/env python
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -15,30 +15,20 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-try:
- from setuptools import setup
-except ImportError:
- from distutils.core import setup
-from sys import version_info
-install_requires = []
-if version_info[:2] <= (2, 5):
- install_requires.append('simplejson >= 2.0.9')
+import setuptools
-setup(
+
+setuptools.setup(
name = 'avro',
version = '@AVRO_VERSION@',
- packages = ['avro',],
+ packages = ['avro'],
package_dir = {'avro': 'src/avro'},
scripts = ["./scripts/avro"],
#include_package_data=True,
package_data={'avro': ['LICENSE', 'NOTICE']},
- # Project uses simplejson, so ensure that it gets installed or upgraded
- # on the target machine
- install_requires = install_requires,
-
# metadata for upload to PyPI
author = 'Apache Avro',
author_email = 'dev@avro.apache.org',
diff --git a/lang/py/src/avro/io.py b/lang/py/src/avro/io.py
index c300fb3..d94f0ff 100644
--- a/lang/py/src/avro/io.py
+++ b/lang/py/src/avro/io.py
@@ -13,6 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
"""
Input/Output utilities, including:
@@ -36,6 +37,7 @@ uses the following mapping:
* Schema doubles are implemented as float.
* Schema booleans are implemented as bool.
"""
+
import struct
from avro import schema
from avro import constants
@@ -47,10 +49,7 @@ import datetime
from decimal import Decimal
from decimal import getcontext
-try:
- import json
-except ImportError:
- import simplejson as json
+import json
#
# Constants
diff --git a/lang/py/src/avro/protocol.py b/lang/py/src/avro/protocol.py
index 48537bd..20ce2d5 100644
--- a/lang/py/src/avro/protocol.py
+++ b/lang/py/src/avro/protocol.py
@@ -13,17 +13,17 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
"""
Protocol implementation.
"""
+
try:
from hashlib import md5
except ImportError:
from md5 import md5
-try:
- import json
-except ImportError:
- import simplejson as json
+
+import json
from avro import schema
#
diff --git a/lang/py/src/avro/schema.py b/lang/py/src/avro/schema.py
index 4661c7e..52684c2 100644
--- a/lang/py/src/avro/schema.py
+++ b/lang/py/src/avro/schema.py
@@ -13,6 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
"""
Contains the Schema classes.
@@ -33,13 +34,11 @@ A schema may be one of:
A boolean; or
Null.
"""
+
from math import floor
from math import log10
-try:
- import json
-except ImportError:
- import simplejson as json
+import json
from avro import constants
diff --git a/lang/py/test/test_script.py b/lang/py/test/test_script.py
index 0ee91be..df02a98 100644
--- a/lang/py/test/test_script.py
+++ b/lang/py/test/test_script.py
@@ -13,13 +13,11 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
import unittest
import csv
from cStringIO import StringIO
-try:
- import json
-except ImportError:
- import simplejson as json
+import json
from tempfile import NamedTemporaryFile
import avro.schema
from avro.io import DatumWriter
[avro] 08/23: AVRO-2445: Remove context manager avoidance (#568)
Posted by rs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rskraba pushed a commit to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/avro.git
commit 138f55f15a6ccbfcab6bcad9b9e6485bb372cf88
Author: Michael A. Smith <mi...@syapse.com>
AuthorDate: Fri Jun 28 04:07:42 2019 -0400
AVRO-2445: Remove context manager avoidance (#568)
---
lang/py/test/test_datafile.py | 8 +-------
1 file changed, 1 insertion(+), 7 deletions(-)
diff --git a/lang/py/test/test_datafile.py b/lang/py/test/test_datafile.py
index 8a81527..102d9b0 100644
--- a/lang/py/test/test_datafile.py
+++ b/lang/py/test/test_datafile.py
@@ -154,13 +154,7 @@ class TestDataFile(unittest.TestCase):
self.assertEquals(correct, len(CODECS_TO_VALIDATE)*len(SCHEMAS_TO_VALIDATE))
def test_context_manager(self):
- # Context manager was introduced as a first class
- # member only in Python 2.6 and above.
- import sys
- if sys.version_info < (2,6):
- print 'Skipping context manager tests on this Python version.'
- return
- # Test the writer with a 'with' statement.
+ """Test the writer with a 'with' statement."""
writer = open(FILENAME, 'wb')
datum_writer = io.DatumWriter()
sample_schema, sample_datum = SCHEMAS_TO_VALIDATE[1]
[avro] 01/23: AVRO-2391 Refactor Py3 Setup (#514)
Posted by rs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rskraba pushed a commit to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/avro.git
commit 6216bba7ae58badd0c6fc835cff312af357981ae
Author: Michael A. Smith <mi...@smith-li.com>
AuthorDate: Fri May 31 13:43:42 2019 -0400
AVRO-2391 Refactor Py3 Setup (#514)
The main purpose of this change is to simplify and modernize the setup code without really changing its behavior.
The main change is that I moved most of the setup code into the declarative setup.cfg, where I hope it is somewhat clearer.
I also fleshed out the setup.py commands so that the build.sh and setup.py have parity -- that is, the build.sh is entirely implemented using setup.py.
---
lang/py3/{build.sh => MANIFEST.in} | 57 ++--------
lang/py3/build.sh | 56 +++-------
lang/py3/setup.cfg | 65 ++++++++++++
lang/py3/setup.py | 209 +++++++++++++++----------------------
4 files changed, 172 insertions(+), 215 deletions(-)
diff --git a/lang/py3/build.sh b/lang/py3/MANIFEST.in
old mode 100755
new mode 100644
similarity index 51%
copy from lang/py3/build.sh
copy to lang/py3/MANIFEST.in
index d28f8cb..acdbd3b
--- a/lang/py3/build.sh
+++ b/lang/py3/MANIFEST.in
@@ -1,5 +1,3 @@
-#!/bin/bash
-
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -7,7 +5,7 @@
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
-# https://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
@@ -15,51 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-set -e # exit on error
-
-function usage {
- echo "Usage: $0 {test|dist|clean}"
- exit 1
-}
-
-if [ $# -eq 0 ]
-then
- usage
-fi
-
-if [ -f VERSION.txt ]
-then
- VERSION=`cat VERSION.txt`
-else
- VERSION=`cat ../../share/VERSION.txt`
-fi
-
-for target in "$@"
-do
-
-function do_clean(){
- python3 setup.py clean
- rm -rvf dist avro_python3.egg-info avro/*.avsc avro/VERSION.txt avro/__pycache__/ avro/tests/interop.avsc avro/tests/__pycache__/
-}
-
-case "$target" in
- test)
- python3 setup.py test
- ;;
-
- dist)
- python3 setup.py sdist
- cp -r dist ../../dist/py3
- ;;
-
- clean)
- do_clean
- ;;
-
- *)
- usage
-esac
-
-done
-
-exit 0
+include avro/HandshakeRequest.avsc
+include avro/HandshakeResponse.avsc
+include avro/VERSION.txt
+include avro/LICENSE
+include avro/NOTICE
diff --git a/lang/py3/build.sh b/lang/py3/build.sh
index d28f8cb..1bb27ad 100755
--- a/lang/py3/build.sh
+++ b/lang/py3/build.sh
@@ -17,49 +17,23 @@
set -e # exit on error
-function usage {
+usage() {
echo "Usage: $0 {test|dist|clean}"
- exit 1
}
-if [ $# -eq 0 ]
-then
- usage
-fi
-
-if [ -f VERSION.txt ]
-then
- VERSION=`cat VERSION.txt`
-else
- VERSION=`cat ../../share/VERSION.txt`
-fi
-
-for target in "$@"
-do
-
-function do_clean(){
- python3 setup.py clean
- rm -rvf dist avro_python3.egg-info avro/*.avsc avro/VERSION.txt avro/__pycache__/ avro/tests/interop.avsc avro/tests/__pycache__/
-}
-
-case "$target" in
- test)
- python3 setup.py test
- ;;
-
- dist)
- python3 setup.py sdist
- cp -r dist ../../dist/py3
- ;;
-
- clean)
- do_clean
- ;;
-
- *)
+main() {
+ local target
+ if (( $# == 0 )); then
usage
-esac
-
-done
+ return 1
+ fi
+ for target; do
+ case "$target" in
+ clean|dist|test) : ;;
+ *) usage; return 1 ;;
+ esac
+ done
+ python3 setup.py "$@"
+}
-exit 0
+main "$@"
diff --git a/lang/py3/setup.cfg b/lang/py3/setup.cfg
new file mode 100644
index 0000000..9d1f437
--- /dev/null
+++ b/lang/py3/setup.cfg
@@ -0,0 +1,65 @@
+##
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+##
+# https://setuptools.readthedocs.io/en/latest/setuptools.html#configuring-setup-using-setup-cfg-files
+[metadata]
+name = avro-python3
+version = file: avro/VERSION.txt
+description = Avro is a serialization and RPC framework.
+long_description = file: README.txt
+keywords =
+ avro
+ serialization
+ rpc
+author = Apache Avro
+author_email = dev@avro.apache.org
+url = http://avro.apache.org/
+license = Apache License 2.0
+classifiers =
+ License :: OSI Approved :: Apache Software License
+ Programming Language :: Python :: 3 :: Only
+ Programming Language :: Python :: 3.4
+ Programming Language :: Python :: 3.5
+ Programming Language :: Python :: 3.6
+ Programming Language :: Python :: 3.7
+ Programming Language :: Python :: 3.8
+
+[options]
+package_dir =
+ avro = avro
+include_package_data = true
+packages = avro
+install_requires =
+zip_safe = true
+scripts =
+ scripts/avro
+python_requires = >=3.4
+test_suite = avro.tests.run_tests
+
+[options.package_data]
+avro =
+ HandshakeRequest.avsc
+ HandshakeResponse.avsc
+ VERSION.txt
+ LICENSE
+ NOTICE
+
+[options.extras_require]
+snappy = snappy
+
+[aliases]
+dist = sdist --dist-dir ../../dist/py3
diff --git a/lang/py3/setup.py b/lang/py3/setup.py
index 5f44dd9..a752779 100755
--- a/lang/py3/setup.py
+++ b/lang/py3/setup.py
@@ -26,133 +26,96 @@ The avro-python3 software is designed for Python 3, but this file and the packag
https://pypi.org/project/avro-python3/
"""
-
+import distutils.command.clean
+import distutils.file_util
+import distutils.dir_util
+import distutils.log
+import fnmatch
import os
-import shutil
-import stat
-import sys
from setuptools import setup
-VERSION_FILE_NAME = 'VERSION.txt'
-LICENSE_FILE_NAME = 'LICENSE'
-NOTICE_FILE_NAME = 'NOTICE'
-
-def RunsFromSourceDist():
- """Tests whether setup.py is invoked from a source distribution.
-
- Returns:
- True if setup.py runs from a source distribution.
- False otherwise, ie. if setup.py runs from the SVN trunk.
- """
- setup_file_path = os.path.abspath(__file__)
- # If a file PKG-INFO exists as a sibling of setup.py,
- # assume we are running as source distribution:
- pkg_info_file_path = \
- os.path.join(os.path.dirname(setup_file_path), 'PKG-INFO')
- return os.path.exists(pkg_info_file_path)
-
-
-def SetupSources():
- """Prepares the source directory.
-
- Runs when setup.py is invoked from the Avro SVN/Git source.
- """
- # Avro lang/py3/ source directory:
- py3_dir = os.path.dirname(os.path.abspath(__file__))
-
- # Avro top-level source directory:
- root_dir = os.path.dirname(os.path.dirname(py3_dir))
-
- # Read and copy Avro version:
- version_file_path = os.path.join(root_dir, 'share', VERSION_FILE_NAME)
- with open(version_file_path, 'r') as f:
- avro_version = f.read().strip()
- shutil.copy(
- src=version_file_path,
- dst=os.path.join(py3_dir, 'avro', VERSION_FILE_NAME),
- )
-
- # Copy necessary avsc files:
- avsc_file_path = os.path.join(
- root_dir, 'share', 'schemas',
- 'org', 'apache', 'avro', 'ipc', 'HandshakeRequest.avsc')
- shutil.copy(
- src=avsc_file_path,
- dst=os.path.join(py3_dir, 'avro', 'HandshakeRequest.avsc'),
- )
-
- avsc_file_path = os.path.join(
- root_dir, 'share', 'schemas',
- 'org', 'apache', 'avro', 'ipc', 'HandshakeResponse.avsc')
- shutil.copy(
- src=avsc_file_path,
- dst=os.path.join(py3_dir, 'avro', 'HandshakeResponse.avsc'),
- )
-
- avsc_file_path = os.path.join(
- root_dir, 'share', 'test', 'schemas', 'interop.avsc')
- shutil.copy(
- src=avsc_file_path,
- dst=os.path.join(py3_dir, 'avro', 'tests', 'interop.avsc'),
- )
-
-
-def ReadVersion():
- """Returns: the content of the Avro version file."""
- setup_file_path = os.path.abspath(__file__)
- install_dir = os.path.dirname(setup_file_path)
- version_file_path = os.path.join(install_dir, 'avro', VERSION_FILE_NAME)
- with open(version_file_path, 'rt') as f:
- avro_version = f.read().strip()
- return avro_version
-
-
-def Main():
- if not RunsFromSourceDist():
- SetupSources()
-
- avro_version = ReadVersion()
-
- setup(
- name = 'avro-python3',
- version = avro_version,
- packages = ['avro'],
- package_dir = {'avro': 'avro'},
- scripts = ['scripts/avro'],
-
- package_data = {
- 'avro': [
- 'HandshakeRequest.avsc',
- 'HandshakeResponse.avsc',
- VERSION_FILE_NAME,
- LICENSE_FILE_NAME,
- NOTICE_FILE_NAME,
- ],
- },
-
- test_suite='avro.tests.run_tests',
- tests_require=[],
-
- # metadata for upload to PyPI
- author = 'Apache Avro',
- author_email = 'dev@avro.apache.org',
- description = 'Avro is a serialization and RPC framework.',
- license = 'Apache License 2.0',
- keywords = 'avro serialization rpc',
- url = 'http://avro.apache.org/',
- classifiers=(
- 'License :: OSI Approved :: Apache Software License',
- 'Programming Language :: Python :: 3 :: Only',
- 'Programming Language :: Python :: 3.4',
- 'Programming Language :: Python :: 3.5',
- 'Programming Language :: Python :: 3.6',
- 'Programming Language :: Python :: 3.7',
- 'Programming Language :: Python :: 3.8',
- ),
- python_requires='>=3.4',
- )
+_HERE = os.path.dirname(os.path.abspath(__file__))
+_AVRO_DIR = os.path.join(_HERE, 'avro')
+_VERSION_FILE_NAME = 'VERSION.txt'
+
+def _is_distribution():
+ """Tests whether setup.py is invoked from a distribution.
+
+ Returns:
+ True if setup.py runs from a distribution.
+ False otherwise, ie. if setup.py runs from a version control work tree.
+ """
+ # If a file PKG-INFO exists as a sibling of setup.py,
+ # assume we are running as source distribution:
+ return os.path.exists(os.path.join(_HERE, 'PKG-INFO'))
+
+
+def _generate_package_data():
+ """Generate package data.
+
+ This data will already exist in a distribution package,
+ so this function only runs for local version control work tree.
+ """
+ distutils.log.info('Generating package data')
+
+ # Avro top-level source directory:
+ root_dir = os.path.dirname(os.path.dirname(_HERE))
+
+ # Create a PEP440 compliant version file.
+ version_file_path = os.path.join(root_dir, 'share', _VERSION_FILE_NAME)
+ with open(version_file_path) as vin:
+ version = vin.read().replace('-', '+')
+ with open(os.path.join(_AVRO_DIR, _VERSION_FILE_NAME), 'w') as vout:
+ vout.write(version)
+
+ # Copy necessary avsc files:
+ avsc_files = (
+ (('schemas', 'org', 'apache', 'avro', 'ipc', 'HandshakeRequest.avsc'), ''),
+ (('schemas', 'org', 'apache', 'avro', 'ipc', 'HandshakeResponse.avsc'), ''),
+ (('test', 'schemas', 'interop.avsc'), ('tests',)),
+ )
+
+ for src, dst in avsc_files:
+ src = os.path.join(root_dir, 'share', *src)
+ dst = os.path.join(_AVRO_DIR, *dst)
+ distutils.file_util.copy_file(src, dst)
+
+
+class CleanCommand(distutils.command.clean.clean):
+ """A command to clean up install artifacts and replaceable, generated files."""
+
+ def _replaceable(self):
+ """Get the list of files to delete."""
+ for name in ('dist', 'avro_python3.egg-info', os.path.join(_AVRO_DIR, _VERSION_FILE_NAME)):
+ if os.path.exists(name):
+ yield name
+ for root, dirnames, filenames in os.walk(_AVRO_DIR):
+ if '__pycache__' in dirnames:
+ dirnames.remove('__pycache__')
+ yield os.path.join(root, '__pycache__')
+ for name in fnmatch.filter(filenames, '*.avsc'):
+ yield os.path.join(root, name)
+
+ def run(self):
+ super().run()
+ for name in self._replaceable():
+ if self.dry_run:
+ distutils.log.info('Would remove %s', name)
+ elif os.path.isdir(name):
+ # distutils logs this for us
+ distutils.dir_util.remove_tree(name)
+ else:
+ distutils.log.info('Removing %s', name)
+ os.remove(name)
+
+
+def main():
+ if not _is_distribution():
+ _generate_package_data()
+
+ setup(cmdclass={"clean": CleanCommand})
if __name__ == '__main__':
- Main()
+ main()
[avro] 04/23: AVRO-1928: Simplify Python float/double encoding
(#528)
Posted by rs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rskraba pushed a commit to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/avro.git
commit 8302424de972a338f3f4d71d224ec1554e653cc3
Author: Michael A. Smith <mi...@smith-li.com>
AuthorDate: Fri May 31 14:38:46 2019 -0400
AVRO-1928: Simplify Python float/double encoding (#528)
Switch to using built-in little endian support in the struct
module, instead of explicit bit manipulation.
---
lang/py/src/avro/io.py | 39 +++++++--------------------------------
lang/py3/avro/io.py | 37 ++++++-------------------------------
2 files changed, 13 insertions(+), 63 deletions(-)
diff --git a/lang/py/src/avro/io.py b/lang/py/src/avro/io.py
index 8c2ef10..c300fb3 100644
--- a/lang/py/src/avro/io.py
+++ b/lang/py/src/avro/io.py
@@ -74,15 +74,14 @@ else:
return struct.unpack(self.format, *args)
struct_class = SimpleStruct
-STRUCT_INT = struct_class('!I') # big-endian unsigned int
-STRUCT_LONG = struct_class('!Q') # big-endian unsigned long long
-STRUCT_FLOAT = struct_class('!f') # big-endian float
-STRUCT_DOUBLE = struct_class('!d') # big-endian double
+STRUCT_FLOAT = struct_class('<f') # big-endian float
+STRUCT_DOUBLE = struct_class('<d') # big-endian double
STRUCT_CRC32 = struct_class('>I') # big-endian unsigned int
STRUCT_SIGNED_SHORT = struct_class('>h') # big-endian signed short
STRUCT_SIGNED_INT = struct_class('>i') # big-endian signed int
STRUCT_SIGNED_LONG = struct_class('>q') # big-endian signed long
+
#
# Exceptions
#
@@ -220,11 +219,7 @@ class BinaryDecoder(object):
The float is converted into a 32-bit integer using a method equivalent to
Java's floatToIntBits and then encoded in little-endian format.
"""
- bits = (((ord(self.read(1)) & 0xffL)) |
- ((ord(self.read(1)) & 0xffL) << 8) |
- ((ord(self.read(1)) & 0xffL) << 16) |
- ((ord(self.read(1)) & 0xffL) << 24))
- return STRUCT_FLOAT.unpack(STRUCT_INT.pack(bits))[0]
+ return STRUCT_FLOAT.unpack(self.read(4))[0]
def read_double(self):
"""
@@ -232,15 +227,7 @@ class BinaryDecoder(object):
The double is converted into a 64-bit integer using a method equivalent to
Java's doubleToLongBits and then encoded in little-endian format.
"""
- bits = (((ord(self.read(1)) & 0xffL)) |
- ((ord(self.read(1)) & 0xffL) << 8) |
- ((ord(self.read(1)) & 0xffL) << 16) |
- ((ord(self.read(1)) & 0xffL) << 24) |
- ((ord(self.read(1)) & 0xffL) << 32) |
- ((ord(self.read(1)) & 0xffL) << 40) |
- ((ord(self.read(1)) & 0xffL) << 48) |
- ((ord(self.read(1)) & 0xffL) << 56))
- return STRUCT_DOUBLE.unpack(STRUCT_LONG.pack(bits))[0]
+ return STRUCT_DOUBLE.unpack(self.read(8))[0]
def read_decimal_from_bytes(self, precision, scale):
"""
@@ -437,11 +424,7 @@ class BinaryEncoder(object):
The float is converted into a 32-bit integer using a method equivalent to
Java's floatToIntBits and then encoded in little-endian format.
"""
- bits = STRUCT_INT.unpack(STRUCT_FLOAT.pack(datum))[0]
- self.write(chr((bits) & 0xFF))
- self.write(chr((bits >> 8) & 0xFF))
- self.write(chr((bits >> 16) & 0xFF))
- self.write(chr((bits >> 24) & 0xFF))
+ self.write(STRUCT_FLOAT.pack(datum))
def write_double(self, datum):
"""
@@ -449,15 +432,7 @@ class BinaryEncoder(object):
The double is converted into a 64-bit integer using a method equivalent to
Java's doubleToLongBits and then encoded in little-endian format.
"""
- bits = STRUCT_LONG.unpack(STRUCT_DOUBLE.pack(datum))[0]
- self.write(chr((bits) & 0xFF))
- self.write(chr((bits >> 8) & 0xFF))
- self.write(chr((bits >> 16) & 0xFF))
- self.write(chr((bits >> 24) & 0xFF))
- self.write(chr((bits >> 32) & 0xFF))
- self.write(chr((bits >> 40) & 0xFF))
- self.write(chr((bits >> 48) & 0xFF))
- self.write(chr((bits >> 56) & 0xFF))
+ self.write(STRUCT_DOUBLE.pack(datum))
def write_decimal_bytes(self, datum, scale):
"""
diff --git a/lang/py3/avro/io.py b/lang/py3/avro/io.py
index cd79393..51f5a13 100644
--- a/lang/py3/avro/io.py
+++ b/lang/py3/avro/io.py
@@ -61,9 +61,8 @@ LONG_MIN_VALUE = -(1 << 63)
LONG_MAX_VALUE = (1 << 63) - 1
STRUCT_INT = struct.Struct('!I') # big-endian unsigned int
-STRUCT_LONG = struct.Struct('!Q') # big-endian unsigned long long
-STRUCT_FLOAT = struct.Struct('!f') # big-endian float
-STRUCT_DOUBLE = struct.Struct('!d') # big-endian double
+STRUCT_FLOAT = struct.Struct('<f') # little-endian float
+STRUCT_DOUBLE = struct.Struct('<d') # little-endian double
STRUCT_CRC32 = struct.Struct('>I') # big-endian unsigned int
@@ -198,11 +197,7 @@ class BinaryDecoder(object):
The float is converted into a 32-bit integer using a method equivalent to
Java's floatToIntBits and then encoded in little-endian format.
"""
- bits = (((ord(self.read(1)) & 0xff)) |
- ((ord(self.read(1)) & 0xff) << 8) |
- ((ord(self.read(1)) & 0xff) << 16) |
- ((ord(self.read(1)) & 0xff) << 24))
- return STRUCT_FLOAT.unpack(STRUCT_INT.pack(bits))[0]
+ return STRUCT_FLOAT.unpack(self.read(4))[0]
def read_double(self):
"""
@@ -210,15 +205,7 @@ class BinaryDecoder(object):
The double is converted into a 64-bit integer using a method equivalent to
Java's doubleToLongBits and then encoded in little-endian format.
"""
- bits = (((ord(self.read(1)) & 0xff)) |
- ((ord(self.read(1)) & 0xff) << 8) |
- ((ord(self.read(1)) & 0xff) << 16) |
- ((ord(self.read(1)) & 0xff) << 24) |
- ((ord(self.read(1)) & 0xff) << 32) |
- ((ord(self.read(1)) & 0xff) << 40) |
- ((ord(self.read(1)) & 0xff) << 48) |
- ((ord(self.read(1)) & 0xff) << 56))
- return STRUCT_DOUBLE.unpack(STRUCT_LONG.pack(bits))[0]
+ return STRUCT_DOUBLE.unpack(self.read(8))[0]
def read_bytes(self):
"""
@@ -341,11 +328,7 @@ class BinaryEncoder(object):
The float is converted into a 32-bit integer using a method equivalent to
Java's floatToIntBits and then encoded in little-endian format.
"""
- bits = STRUCT_INT.unpack(STRUCT_FLOAT.pack(datum))[0]
- self.WriteByte((bits) & 0xFF)
- self.WriteByte((bits >> 8) & 0xFF)
- self.WriteByte((bits >> 16) & 0xFF)
- self.WriteByte((bits >> 24) & 0xFF)
+ self.write(STRUCT_FLOAT.pack(datum))
def write_double(self, datum):
"""
@@ -353,15 +336,7 @@ class BinaryEncoder(object):
The double is converted into a 64-bit integer using a method equivalent to
Java's doubleToLongBits and then encoded in little-endian format.
"""
- bits = STRUCT_LONG.unpack(STRUCT_DOUBLE.pack(datum))[0]
- self.WriteByte((bits) & 0xFF)
- self.WriteByte((bits >> 8) & 0xFF)
- self.WriteByte((bits >> 16) & 0xFF)
- self.WriteByte((bits >> 24) & 0xFF)
- self.WriteByte((bits >> 32) & 0xFF)
- self.WriteByte((bits >> 40) & 0xFF)
- self.WriteByte((bits >> 48) & 0xFF)
- self.WriteByte((bits >> 56) & 0xFF)
+ self.write(STRUCT_DOUBLE.pack(datum))
def write_bytes(self, datum):
"""
[avro] 13/23: AVRO-2578: Deprecate Parse in 1.9.2.
Posted by rs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rskraba pushed a commit to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/avro.git
commit 1c190fc0a44a0da443795d8317fd5c8ff5bf1b95
Author: Ryan Skraba <ry...@skraba.com>
AuthorDate: Tue Jan 21 14:42:41 2020 +0100
AVRO-2578: Deprecate Parse in 1.9.2.
---
lang/py3/avro/protocol.py | 2 +-
lang/py3/avro/schema.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/lang/py3/avro/protocol.py b/lang/py3/avro/protocol.py
index 8ee721c..2cb4f7c 100644
--- a/lang/py3/avro/protocol.py
+++ b/lang/py3/avro/protocol.py
@@ -399,7 +399,7 @@ def parse(json_string):
def Parse(json_string):
"""Deprecated implementation of parse."""
- warnings.warn("`Parse` is deprecated in avro 1.10. "
+ warnings.warn("`Parse` is deprecated in avro 1.9.2. "
"Please use `parse` (lowercase) instead.",
DeprecationWarning)
return parse(json_string)
diff --git a/lang/py3/avro/schema.py b/lang/py3/avro/schema.py
index 2bfaa3b..e76d7be 100644
--- a/lang/py3/avro/schema.py
+++ b/lang/py3/avro/schema.py
@@ -1246,7 +1246,7 @@ def parse(json_string):
def Parse(json_string):
"""Deprecated implementation of parse."""
- warnings.warn("`Parse` is deprecated in avro 1.10. "
+ warnings.warn("`Parse` is deprecated in avro 1.9.2. "
"Please use `parse` (lowercase) instead.",
DeprecationWarning)
return parse(json_string)
[avro] 23/23: AVRO-2595: Remove Unusable Py3 Txipc (#716)
Posted by rs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rskraba pushed a commit to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/avro.git
commit 4006a7643c28a658bf2585761f64b4095aa51a71
Author: Michael A. Smith <mi...@smith-li.com>
AuthorDate: Sun Dec 1 06:18:58 2019 -0500
AVRO-2595: Remove Unusable Py3 Txipc (#716)
The Py3 package will be superseded by python
---
lang/py3/avro/tests/txsample_http_client.py | 106 -------------
lang/py3/avro/tests/txsample_http_server.py | 71 ---------
lang/py3/avro/txipc.py | 223 ----------------------------
3 files changed, 400 deletions(-)
diff --git a/lang/py3/avro/tests/txsample_http_client.py b/lang/py3/avro/tests/txsample_http_client.py
deleted file mode 100644
index a536aa8..0000000
--- a/lang/py3/avro/tests/txsample_http_client.py
+++ /dev/null
@@ -1,106 +0,0 @@
-#!/usr/bin/env python3
-# -*- mode: python -*-
-# -*- coding: utf-8 -*-
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import sys
-
-from avro import protocol, txipc
-from twisted.internet import defer, reactor
-from twisted.python.util import println
-
-MAIL_PROTOCOL_JSON = """\
-{"namespace": "example.proto",
- "protocol": "Mail",
-
- "types": [
- {"name": "Message", "type": "record",
- "fields": [
- {"name": "to", "type": "string"},
- {"name": "from", "type": "string"},
- {"name": "body", "type": "string"}
- ]
- }
- ],
-
- "messages": {
- "send": {
- "request": [{"name": "message", "type": "Message"}],
- "response": "string"
- },
- "replay": {
- "request": [],
- "response": "string"
- }
- }
-}
-"""
-MAIL_PROTOCOL = protocol.parse(MAIL_PROTOCOL_JSON)
-SERVER_HOST = 'localhost'
-SERVER_PORT = 9090
-
-class UsageError(Exception):
- def __init__(self, value):
- self.value = value
- def __str__(self):
- return repr(self.value)
-
-def make_requestor(server_host, server_port, protocol):
- client = txipc.TwistedHTTPTransceiver(SERVER_HOST, SERVER_PORT)
- return txipc.TwistedRequestor(protocol, client)
-
-if __name__ == '__main__':
- if len(sys.argv) not in [4, 5]:
- raise UsageError("Usage: <to> <from> <body> [<count>]")
-
- # client code - attach to the server and send a message
- # fill in the Message record
- message = dict()
- message['to'] = sys.argv[1]
- message['from'] = sys.argv[2]
- message['body'] = sys.argv[3]
-
- try:
- num_messages = int(sys.argv[4])
- except IndexError:
- num_messages = 1
-
- # build the parameters for the request
- params = {}
- params['message'] = message
-
- requests = []
- # send the requests and print the result
- for msg_count in range(num_messages):
- requestor = make_requestor(SERVER_HOST, SERVER_PORT, MAIL_PROTOCOL)
- d = requestor.request('send', params)
- d.addCallback(lambda result: println("Result: " + result))
- requests.append(d)
- results = defer.gatherResults(requests)
-
- def replay_cb(result):
- print("Replay Result: " + result)
- reactor.stop()
-
- def replay(_):
- # try out a replay message
- requestor = make_requestor(SERVER_HOST, SERVER_PORT, MAIL_PROTOCOL)
- d = requestor.request('replay', dict())
- d.addCallback(replay_cb)
-
- results.addCallback(replay)
- reactor.run()
diff --git a/lang/py3/avro/tests/txsample_http_server.py b/lang/py3/avro/tests/txsample_http_server.py
deleted file mode 100644
index beb1267..0000000
--- a/lang/py3/avro/tests/txsample_http_server.py
+++ /dev/null
@@ -1,71 +0,0 @@
-#!/usr/bin/env python3
-# -*- mode: python -*-
-# -*- coding: utf-8 -*-
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-
-
-from avro import ipc, protocol, txipc
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-from twisted.internet import reactor
-# See the License for the specific language governing permissions and
-# limitations under the License.
-from twisted.web import server
-
-MAIL_PROTOCOL_JSON = """\
-{"namespace": "example.proto",
- "protocol": "Mail",
-
- "types": [
- {"name": "Message", "type": "record",
- "fields": [
- {"name": "to", "type": "string"},
- {"name": "from", "type": "string"},
- {"name": "body", "type": "string"}
- ]
- }
- ],
-
- "messages": {
- "send": {
- "request": [{"name": "message", "type": "Message"}],
- "response": "string"
- },
- "replay": {
- "request": [],
- "response": "string"
- }
- }
-}
-"""
-MAIL_PROTOCOL = protocol.parse(MAIL_PROTOCOL_JSON)
-SERVER_ADDRESS = ('localhost', 9090)
-
-class MailResponder(ipc.Responder):
- def __init__(self):
- ipc.Responder.__init__(self, MAIL_PROTOCOL)
-
- def invoke(self, message, request):
- if message.name == 'send':
- request_content = request['message']
- response = "Sent message to %(to)s from %(from)s with body %(body)s" % \
- request_content
- return response
- elif message.name == 'replay':
- return 'replay'
-
-if __name__ == '__main__':
- root = server.Site(txipc.AvroResponderResource(MailResponder()))
- reactor.listenTCP(9090, root)
- reactor.run()
diff --git a/lang/py3/avro/txipc.py b/lang/py3/avro/txipc.py
deleted file mode 100644
index d8e3225..0000000
--- a/lang/py3/avro/txipc.py
+++ /dev/null
@@ -1,223 +0,0 @@
-#!/usr/bin/env python3
-# -*- mode: python -*-
-# -*- coding: utf-8 -*-
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import io
-
-from zope.interface import implements
-
-from avro import io as avro_io
-from avro import ipc
-from twisted.internet.defer import Deferred, maybeDeferred
-from twisted.internet.protocol import Protocol
-from twisted.web import resource, server
-from twisted.web.client import Agent
-from twisted.web.http_headers import Headers
-from twisted.web.iweb import IBodyProducer
-
-
-class TwistedRequestor(ipc.BaseRequestor):
- """A Twisted-compatible requestor. Returns a Deferred that will fire with the
- returning value, instead of blocking until the request completes."""
- def _process_handshake(self, call_response, message_name, request_datum):
- # process the handshake and call response
- buffer_decoder = avro_io.BinaryDecoder(io.StringIO(call_response))
- call_response_exists = self.read_handshake_response(buffer_decoder)
- if call_response_exists:
- return self.read_call_response(message_name, buffer_decoder)
- else:
- return self.request(message_name, request_datum)
-
- def issue_request(self, call_request, message_name, request_datum):
- d = self.transceiver.transceive(call_request)
- d.addCallback(self._process_handshake, message_name, request_datum)
- return d
-
-class RequestStreamingProducer(object):
- """A streaming producer for issuing requests with the Twisted.web Agent."""
- implements(IBodyProducer)
-
- paused = False
- stopped = False
- started = False
-
- def __init__(self, message):
- self._message = message
- self._length = len(message)
- # We need a buffer length header for every buffer and an additional
- # zero-length buffer as the message terminator
- self._length += (self._length / ipc.BUFFER_SIZE + 2) \
- * ipc.BUFFER_HEADER_LENGTH
- self._total_bytes_sent = 0
- self._deferred = Deferred()
-
- # read-only properties
- message = property(lambda self: self._message)
- length = property(lambda self: self._length)
- consumer = property(lambda self: self._consumer)
- deferred = property(lambda self: self._deferred)
-
- def _get_total_bytes_sent(self):
- return self._total_bytes_sent
-
- def _set_total_bytes_sent(self, bytes_sent):
- self._total_bytes_sent = bytes_sent
-
- total_bytes_sent = property(_get_total_bytes_sent, _set_total_bytes_sent)
-
- def startProducing(self, consumer):
- if self.started:
- return
-
- self.started = True
- self._consumer = consumer
- # Keep writing data to the consumer until we're finished,
- # paused (pauseProducing()) or stopped (stopProducing())
- while self.length - self.total_bytes_sent > 0 and \
- not self.paused and not self.stopped:
- self.write()
- # self.write will fire this deferred once it has written
- # the entire message to the consumer
- return self.deferred
-
- def resumeProducing(self):
- self.paused = False
- self.write(self)
-
- def pauseProducing(self):
- self.paused = True
-
- def stopProducing(self):
- self.stopped = True
-
- def write(self):
- if self.length - self.total_bytes_sent > ipc.BUFFER_SIZE:
- buffer_length = ipc.BUFFER_SIZE
- else:
- buffer_length = self.length - self.total_bytes_sent
- self.write_buffer(self.message[self.total_bytes_sent:
- (self.total_bytes_sent + buffer_length)])
- self.total_bytes_sent += buffer_length
- # Make sure we wrote the entire message
- if self.total_bytes_sent == self.length and not self.stopped:
- self.stopProducing()
- # A message is always terminated by a zero-length buffer.
- self.write_buffer_length(0)
- self.deferred.callback(None)
-
- def write_buffer(self, chunk):
- buffer_length = len(chunk)
- self.write_buffer_length(buffer_length)
- self.consumer.write(chunk)
-
- def write_buffer_length(self, n):
- self.consumer.write(ipc.BIG_ENDIAN_INT_STRUCT.pack(n))
-
-class AvroProtocol(Protocol):
-
- recvd = ''
- done = False
-
- def __init__(self, finished):
- self.finished = finished
- self.message = []
-
- def dataReceived(self, data):
- self.recvd = self.recvd + data
- while len(self.recvd) >= ipc.BUFFER_HEADER_LENGTH:
- buffer_length ,= ipc.BIG_ENDIAN_INT_STRUCT.unpack(
- self.recvd[:ipc.BUFFER_HEADER_LENGTH])
- if buffer_length == 0:
- response = ''.join(self.message)
- self.done = True
- self.finished.callback(response)
- break
- if len(self.recvd) < buffer_length + ipc.BUFFER_HEADER_LENGTH:
- break
- buffer = self.recvd[ipc.BUFFER_HEADER_LENGTH:buffer_length + ipc.BUFFER_HEADER_LENGTH]
- self.recvd = self.recvd[buffer_length + ipc.BUFFER_HEADER_LENGTH:]
- self.message.append(buffer)
-
- def connectionLost(self, reason):
- if not self.done:
- self.finished.errback(ipc.ConnectionClosedException("Reader read 0 bytes."))
-
-class TwistedHTTPTransceiver(object):
- """This transceiver uses the Agent class present in Twisted.web >= 9.0
- for issuing requests to the remote endpoint."""
- def __init__(self, host, port, remote_name=None, reactor=None):
- self.url = "http://%s:%d/" % (host, port)
-
- if remote_name is None:
- # There's no easy way to get this peer's remote address
- # in Twisted so I use a random UUID to identify ourselves
- import uuid
- self.remote_name = uuid.uuid4()
-
- if reactor is None:
- from twisted.internet import reactor
- self.agent = Agent(reactor)
-
- def read_framed_message(self, response):
- finished = Deferred()
- response.deliverBody(AvroProtocol(finished))
- return finished
-
- def transceive(self, request):
- req_method = 'POST'
- req_headers = {
- 'Content-Type': ['avro/binary'],
- 'Accept-Encoding': ['identity'],
- }
-
- body_producer = RequestStreamingProducer(request)
- d = self.agent.request(
- req_method,
- self.url,
- headers=Headers(req_headers),
- bodyProducer=body_producer)
- return d.addCallback(self.read_framed_message)
-
-class AvroResponderResource(resource.Resource):
- """This Twisted.web resource can be placed anywhere in a URL hierarchy
- to provide an Avro endpoint. Different Avro protocols can be served
- by the same web server as long as they are in different resources in
- a URL hierarchy."""
- isLeaf = True
-
- def __init__(self, responder):
- resource.Resource.__init__(self)
- self.responder = responder
-
- def cb_render_POST(self, resp_body, request):
- request.setResponseCode(200)
- request.setHeader('Content-Type', 'avro/binary')
- resp_writer = ipc.FramedWriter(request)
- resp_writer.write_framed_message(resp_body)
- request.finish()
-
- def render_POST(self, request):
- # Unfortunately, Twisted.web doesn't support incoming
- # streamed input yet, the whole payload must be kept in-memory
- request.content.seek(0, 0)
- call_request_reader = ipc.FramedReader(request.content)
- call_request = call_request_reader.read_framed_message()
- d = maybeDeferred(self.responder.respond, call_request)
- d.addCallback(self.cb_render_POST, request)
- return server.NOT_DONE_YET
[avro] 20/23: AVRO-2603: Clean Up Imports (#684)
Posted by rs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rskraba pushed a commit to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/avro.git
commit 76611fcf44b5b9330213163330195a1a82c24d67
Author: Michael A. Smith <mi...@smith-li.com>
AuthorDate: Fri Oct 25 08:59:58 2019 -0400
AVRO-2603: Clean Up Imports (#684)
---
lang/py/test/test_tether_word_count.py | 35 +++++++++++-----------------------
1 file changed, 11 insertions(+), 24 deletions(-)
diff --git a/lang/py/test/test_tether_word_count.py b/lang/py/test/test_tether_word_count.py
index 8c3fb08..38cd2d9 100644
--- a/lang/py/test/test_tether_word_count.py
+++ b/lang/py/test/test_tether_word_count.py
@@ -19,13 +19,18 @@
from __future__ import absolute_import, division, print_function
-import inspect
import os
+import shutil
import subprocess
import sys
-import time
+import tempfile
import unittest
+import avro
+import avro.datafile
+import avro.io
+import avro.schema
+import avro.tether.tether_task_runner
import set_avro_test_path
@@ -41,10 +46,6 @@ class TestTetherWordCount(unittest.TestCase):
lines - list of strings to write
fname - the name of the file to write to.
"""
- import avro.io as avio
- from avro.datafile import DataFileReader,DataFileWriter
- from avro import schema
-
#recursively make all directories
dparts=fname.split(os.sep)[:-1]
for i in range(len(dparts)):
@@ -55,10 +56,7 @@ class TestTetherWordCount(unittest.TestCase):
with file(fname,'w') as hf:
inschema="""{"type":"string"}"""
- writer=DataFileWriter(hf,avio.DatumWriter(inschema),writers_schema=schema.parse(inschema))
-
- #encoder = avio.BinaryEncoder(writer)
- #datum_writer = avio.DatumWriter()
+ writer = avro.datafile.DataFileWriter(hf, avro.io.DatumWriter(inschema), writers_schema=avro.schema.parse(inschema))
for datum in lines:
writer.append(datum)
@@ -89,17 +87,6 @@ class TestTetherWordCount(unittest.TestCase):
Assumptions: 1) bash is available in /bin/bash
"""
- from word_count_task import WordCountTask
- from avro.tether import tether_task_runner
- from avro.datafile import DataFileReader
- from avro.io import DatumReader
- import avro
-
- import subprocess
- import shutil
- import tempfile
- import inspect
-
proc=None
exfile = None
@@ -163,7 +150,7 @@ class TestTetherWordCount(unittest.TestCase):
# form the arguments for the subprocess
subargs=[]
- srcfile=inspect.getsourcefile(tether_task_runner)
+ srcfile = avro.tether.tether_task_runner.__file__
# Create a shell script to act as the program we want to execute
# We do this so we can set the python path appropriately
@@ -173,7 +160,7 @@ python -m avro.tether.tether_task_runner word_count_task.WordCountTask
"""
# We need to make sure avro is on the path
# getsourcefile(avro) returns .../avro/__init__.py
- asrc=inspect.getsourcefile(avro)
+ asrc = avro.__file__
apath=asrc.rsplit(os.sep,2)[0]
# path to where the tests lie
@@ -197,7 +184,7 @@ python -m avro.tether.tether_task_runner word_count_task.WordCountTask
# read the output
with file(os.path.join(outpath,"part-00000.avro")) as hf:
- reader=DataFileReader(hf, DatumReader())
+ reader = avro.datafile.DataFileReader(hf, avro.io.DatumReader())
for record in reader:
self.assertEqual(record["value"],true_counts[record["key"]])
[avro] 05/23: AVRO-2104: Schema normalization and fingerprinting
for Python 3 (#536)
Posted by rs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rskraba pushed a commit to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/avro.git
commit a6279592f078bfff9ca38fed0e5672fbaafcf1eb
Author: Michael A. Smith <mi...@syapse.com>
AuthorDate: Mon Jun 3 10:37:03 2019 -0400
AVRO-2104: Schema normalization and fingerprinting for Python 3 (#536)
* AVRO-2104: Schema normalization and fingerprinting for Python 3.
An implementation of schema normalization and fingerprinting for Python
3. The code is a fairly literal translation of the Java code and is
entirely additional to the existing code, except for one small change
in the way schema.Name is implemented to improve compatibility with the
Java version.
The new implementation is covered by 138 unit tests which have been
automatically generated by a script (not provided) which executed the
Java Avro normalization and fingerprinting code to provide expected
values for the Python implementation to reproduce.
* AVRO-2104: Include License Header
Closes #258
Closes #536
---
lang/py3/avro/schema.py | 11 +-
lang/py3/avro/schemanormalization.py | 224 ++++
lang/py3/avro/tests/test_normalization.py | 2029 +++++++++++++++++++++++++++++
3 files changed, 2259 insertions(+), 5 deletions(-)
diff --git a/lang/py3/avro/schema.py b/lang/py3/avro/schema.py
index 1530530..b0b6c4c 100644
--- a/lang/py3/avro/schema.py
+++ b/lang/py3/avro/schema.py
@@ -41,7 +41,6 @@ A schema may be one of:
from types import MappingProxyType
import abc
-import collections
import json
import logging
import re
@@ -271,7 +270,9 @@ class Name(object):
# name is relative, combine with explicit namespace:
self._name = name
self._namespace = namespace
- self._fullname = '%s.%s' % (self._namespace, self._name)
+ self._fullname = (self._name
+ if (not self._namespace) else
+ '%s.%s' % (self._namespace, self._name))
# Validate the fullname:
if _RE_FULL_NAME.match(self._fullname) is None:
@@ -281,8 +282,8 @@ class Name(object):
def __eq__(self, other):
if not isinstance(other, Name):
- return False
- return (self.fullname == other.fullname)
+ return NotImplemented
+ return self.fullname == other.fullname
@property
def simple_name(self):
@@ -296,7 +297,7 @@ class Name(object):
@property
def fullname(self):
- """Returns: the full name (always contains a period '.')."""
+ """Returns: the full name."""
return self._fullname
diff --git a/lang/py3/avro/schemanormalization.py b/lang/py3/avro/schemanormalization.py
new file mode 100644
index 0000000..9e6d364
--- /dev/null
+++ b/lang/py3/avro/schemanormalization.py
@@ -0,0 +1,224 @@
+#!/usr/bin/env python
+# -*- mode: python -*-
+# -*- coding: utf-8 -*-
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import hashlib
+from io import StringIO
+
+from avro.schema import (ERROR, RECORD, UNION, ARRAY, MAP, ENUM, FIXED)
+
+
+def ToParsingCanonicalForm(schema):
+ """Returns the "Parsing Canonical Form" of a schema.
+
+ The Parsing Canonical Form is defined by the Avro specification.
+
+ Args:
+ schema: The Schema to be normalized.
+ Returns:
+ A string containing the canonical JSON schema.
+ """
+
+ env = {}
+ with StringIO() as output:
+ return _BuildCanonicalForm(env, schema, output).getvalue()
+
+
+def _BuildCanonicalForm(env, s, o):
+ first_time = True
+ st = s.type
+
+ # The Java Avro implementation represents error records as records with
+ # a separate error flag. For canonicalization to be consistent across
+ # implementations we must normalize errors to have record type here.
+ if st == ERROR:
+ st = RECORD
+
+ if st == UNION:
+ o.write('[')
+ for b in s.schemas:
+ if not first_time:
+ o.write(',')
+ else:
+ first_time = False
+ _BuildCanonicalForm(env, b, o)
+ o.write(']')
+ return o
+
+ elif st in {ARRAY, MAP}:
+ o.write('{"type":"')
+ o.write(st)
+ o.write('"')
+ if st == ARRAY:
+ o.write(',"items":')
+ _BuildCanonicalForm(env, s.items, o)
+ else:
+ o.write(',"values":')
+ _BuildCanonicalForm(env, s.values, o)
+ o.write('}')
+ return o
+
+ elif st in {ENUM, FIXED, RECORD}:
+ name = s.fullname
+ if name in env:
+ o.write(env[name])
+ return o
+ qname = '"%s"' % name
+ env[name] = qname
+ o.write('{"name":')
+ o.write(qname)
+ o.write(',"type":"')
+ o.write(st)
+ o.write('"')
+ if st == ENUM:
+ o.write(',"symbols":[')
+ for enum_symbol in s.symbols:
+ if not first_time:
+ o.write(',')
+ else:
+ first_time = False
+ o.write('"')
+ o.write(enum_symbol)
+ o.write('"')
+ o.write("]")
+ elif st == FIXED:
+ o.write(',"size":')
+ o.write(str(s.size))
+ else: # st == RECORD or st == ERROR
+ o.write(',"fields":[')
+ for f in s.fields:
+ if not first_time:
+ o.write(',')
+ else:
+ first_time = False
+ o.write('{"name":"')
+ o.write(f.name)
+ o.write('"')
+ o.write(',"type":')
+ _BuildCanonicalForm(env, f.type, o)
+ o.write('}')
+ o.write(']')
+ o.write('}')
+ return o
+ else:
+ # boolean, bytes, double, float, int, long, null, string
+ o.write('"')
+ o.write(st)
+ o.write('"')
+ return o
+
+
+def Fingerprint(parsing_normal_form_schema, fingerprint_algorithm_name):
+ """Returns a fingerprint of a string of bytes.
+
+ Args:
+ parsing_normal_form_schema: A string containing an Avro
+ schema in parsing normal form, such as one obtained
+ by passing a schema object to ToParsingCanonicalForm()
+ fingerprint_algorithm_name: One of the algorithm names
+ returned by FingerprintAlgorithmNames(), typically
+ 'CRC-64-AVRO', 'md5' or 'sha256'. See the Avro
+ specification for guidance on selecting a
+ fingerprinting algorithm.
+ Returns:
+ A bytes object containing the schema fingerprint.
+ """
+ if fingerprint_algorithm_name not in FingerprintAlgorithmNames():
+ raise ValueError("Unknown schema fingerprint algorithm {!r}"
+ .format(fingerprint_algorithm_name))
+ fingerprint_algorithm = _FINGERPRINT_ALIASES_TO_NAMES[fingerprint_algorithm_name]
+ data = parsing_normal_form_schema.encode('utf-8')
+ if fingerprint_algorithm in _CRC_64_AVRO:
+ return _Crc64AvroFingerprint(data)
+ h = hashlib.new(fingerprint_algorithm, data)
+ return h.digest()
+
+
+_CRC_64_AVRO = frozenset({'CRC-64-AVRO'})
+_PYTHON_DIGEST_NAMES = frozenset(hashlib.algorithms_guaranteed | _CRC_64_AVRO)
+
+# These are the only three algorithms which Java implementations are
+# *required* to support. We provide aliases for them here when the
+# Python implementation also supports them, so that key fingerprint
+# algorithm names can interoperate between Java and Python.
+_JAVA_TO_PYTHON_DIGEST_NAMES = {
+ 'MD5': 'md5',
+ 'SHA-1': 'sha1',
+ 'SHA-256': 'sha256'}
+
+_AVAILABLE_JAVA_TO_PYTHON_DIGEST_NAMES = {
+ j: p for j, p in _JAVA_TO_PYTHON_DIGEST_NAMES.items()
+ if p in hashlib.algorithms_guaranteed}
+
+_FINGERPRINT_ALIASES_TO_NAMES = {name: name for name in _PYTHON_DIGEST_NAMES}
+_FINGERPRINT_ALIASES_TO_NAMES.update(_AVAILABLE_JAVA_TO_PYTHON_DIGEST_NAMES)
+
+
+def FingerprintAlgorithmNames():
+ """A collection of fingerprint algorithm names.
+
+ The same algorithm may be associated with more than one entry
+ in this collection. For example, 'SHA-1' and 'sha1' might both
+ be present in the result, in order to facilitate the interoperability
+ of algorithm names between Python and, say, Java Avro
+ implementations.
+
+ Returns:
+ A set of strings containing algorithm names, any
+ one of which can be used as the fingerprint_algorithm
+ argument of Fingerprint()
+ """
+ return _FINGERPRINT_ALIASES_TO_NAMES.keys()
+
+
+_EMPTY64 = 0xc15d213aa4d7a795
+
+
+def _Crc64AvroFingerprint(data):
+ """The 64-bit Rabin Fingerprint.
+
+ As described in the Avro specification.
+
+ Args:
+ data: A bytes object containing the UTF-8 encoded parsing canonical
+ form of an Avro schema.
+ Returns:
+ A bytes object with a length of eight.
+ """
+ if _FP_TABLE is None:
+ _PopulateFpTable()
+ result = _EMPTY64
+ for b in data:
+ result = (result >> 8) ^ _FP_TABLE[(result ^ b) & 0xff]
+ # Although not mentioned in the Avro specification, the Java
+ # implementation gives fingerprint bytes in little-endian order
+ return result.to_bytes(length=8, byteorder='little', signed=False)
+
+_FP_TABLE = None
+
+
+def _PopulateFpTable():
+ global _FP_TABLE
+ _FP_TABLE = []
+ for i in range(256):
+ fp = i
+ for j in range(8):
+ mask = -(fp & 1)
+ fp = (fp >> 1) ^ (_EMPTY64 & mask)
+ _FP_TABLE.append(fp)
diff --git a/lang/py3/avro/tests/test_normalization.py b/lang/py3/avro/tests/test_normalization.py
new file mode 100644
index 0000000..8935529
--- /dev/null
+++ b/lang/py3/avro/tests/test_normalization.py
@@ -0,0 +1,2029 @@
+#!/usr/bin/env python
+# -*- mode: python -*-
+# -*- coding: utf-8 -*-
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from avro.schema import Parse
+from avro.schemanormalization import ToParsingCanonicalForm, Fingerprint, FingerprintAlgorithmNames
+
+
+class TestSchemaNormalization(unittest.TestCase):
+
+ def testCanonicalization1(self):
+ pre='"float"'
+ post='"float"'
+ pre_schema = Parse(pre)
+ canonical_form = ToParsingCanonicalForm(pre_schema)
+ self.assertEqual(canonical_form, post)
+
+ def testCanonicalization2(self):
+ pre='{"type": "float"}'
+ post='"float"'
+ pre_schema = Parse(pre)
+ canonical_form = ToParsingCanonicalForm(pre_schema)
+ self.assertEqual(canonical_form, post)
+
+ def testCanonicalization3(self):
+ pre='"int"'
+ post='"int"'
+ pre_schema = Parse(pre)
+ canonical_form = ToParsingCanonicalForm(pre_schema)
+ self.assertEqual(canonical_form, post)
+
+ def testCanonicalization4(self):
+ pre='{"type": "int"}'
+ post='"int"'
+ pre_schema = Parse(pre)
+ canonical_form = ToParsingCanonicalForm(pre_schema)
+ self.assertEqual(canonical_form, post)
+
+ def testCanonicalization5(self):
+ pre='"double"'
+ post='"double"'
+ pre_schema = Parse(pre)
+ canonical_form = ToParsingCanonicalForm(pre_schema)
+ self.assertEqual(canonical_form, post)
+
+ def testCanonicalization6(self):
+ pre='{"type": "double"}'
+ post='"double"'
+ pre_schema = Parse(pre)
+ canonical_form = ToParsingCanonicalForm(pre_schema)
+ self.assertEqual(canonical_form, post)
+
+ def testCanonicalization7(self):
+ pre='"null"'
+ post='"null"'
+ pre_schema = Parse(pre)
+ canonical_form = ToParsingCanonicalForm(pre_schema)
+ self.assertEqual(canonical_form, post)
+
+ def testCanonicalization8(self):
+ pre='{"type": "null"}'
+ post='"null"'
+ pre_schema = Parse(pre)
+ canonical_form = ToParsingCanonicalForm(pre_schema)
+ self.assertEqual(canonical_form, post)
+
+ def testCanonicalization9(self):
+ pre='"bytes"'
+ post='"bytes"'
+ pre_schema = Parse(pre)
+ canonical_form = ToParsingCanonicalForm(pre_schema)
+ self.assertEqual(canonical_form, post)
+
+ def testCanonicalization10(self):
+ pre='{"type": "bytes"}'
+ post='"bytes"'
+ pre_schema = Parse(pre)
+ canonical_form = ToParsingCanonicalForm(pre_schema)
+ self.assertEqual(canonical_form, post)
+
+ def testCanonicalization11(self):
+ pre='"long"'
+ post='"long"'
+ pre_schema = Parse(pre)
+ canonical_form = ToParsingCanonicalForm(pre_schema)
+ self.assertEqual(canonical_form, post)
+
+ def testCanonicalization12(self):
+ pre='{"type": "long"}'
+ post='"long"'
+ pre_schema = Parse(pre)
+ canonical_form = ToParsingCanonicalForm(pre_schema)
+ self.assertEqual(canonical_form, post)
+
+ def testCanonicalization13(self):
+ pre='"boolean"'
+ post='"boolean"'
+ pre_schema = Parse(pre)
+ canonical_form = ToParsingCanonicalForm(pre_schema)
+ self.assertEqual(canonical_form, post)
+
+ def testCanonicalization14(self):
+ pre='{"type": "boolean"}'
+ post='"boolean"'
+ pre_schema = Parse(pre)
+ canonical_form = ToParsingCanonicalForm(pre_schema)
+ self.assertEqual(canonical_form, post)
+
+ def testCanonicalization15(self):
+ pre='"string"'
+ post='"string"'
+ pre_schema = Parse(pre)
+ canonical_form = ToParsingCanonicalForm(pre_schema)
+ self.assertEqual(canonical_form, post)
+
+ def testCanonicalization16(self):
+ pre='{"type": "string"}'
+ post='"string"'
+ pre_schema = Parse(pre)
+ canonical_form = ToParsingCanonicalForm(pre_schema)
+ self.assertEqual(canonical_form, post)
+
+ def testCanonicalization17(self):
+ pre='{"type": "fixed", "name": "Test", "size": 1}'
+ post='{"name":"Test","type":"fixed","size":1}'
+ pre_schema = Parse(pre)
+ canonical_form = ToParsingCanonicalForm(pre_schema)
+ self.assertEqual(canonical_form, post)
+
+ def testCanonicalization18(self):
+ pre="""
+ {
+ "type": "fixed",
+ "name": "MyFixed",
+ "namespace": "org.apache.hadoop.avro",
+ "size": 1
+ }
+ """
+ post='{"name":"org.apache.hadoop.avro.MyFixed","type":"fixed","size":1}'
+ pre_schema = Parse(pre)
+ canonical_form = ToParsingCanonicalForm(pre_schema)
+ self.assertEqual(canonical_form, post)
+
+ def testCanonicalization19(self):
+ pre='{"type": "enum", "name": "Test", "symbols": ["A", "B"]}'
+ post='{"name":"Test","type":"enum","symbols":["A","B"]}'
+ pre_schema = Parse(pre)
+ canonical_form = ToParsingCanonicalForm(pre_schema)
+ self.assertEqual(canonical_form, post)
+
+ def testCanonicalization20(self):
+ pre='{"type": "array", "items": "long"}'
+ post='{"type":"array","items":"long"}'
+ pre_schema = Parse(pre)
+ canonical_form = ToParsingCanonicalForm(pre_schema)
+ self.assertEqual(canonical_form, post)
+
+ def testCanonicalization21(self):
+ pre="""
+ {
+ "type": "array",
+ "items": {"type": "enum", "name": "Test", "symbols": ["A", "B"]}
+ }
+ """
+ post=('{"type":"array","items":{"name":"Test'
+ '","type":"enum","symbols":["A","B"]}}')
+ pre_schema = Parse(pre)
+ canonical_form = ToParsingCanonicalForm(pre_schema)
+ self.assertEqual(canonical_form, post)
+
+ def testCanonicalization22(self):
+ pre='{"type": "map", "values": "long"}'
+ post='{"type":"map","values":"long"}'
+ pre_schema = Parse(pre)
+ canonical_form = ToParsingCanonicalForm(pre_schema)
+ self.assertEqual(canonical_form, post)
+
+ def testCanonicalization23(self):
+ pre="""
+ {
+ "type": "map",
+ "values": {"type": "enum", "name": "Test", "symbols": ["A", "B"]}
+ }
+ """
+ post=('{"type":"map","values":{"name":"Test"'
+ ',"type":"enum","symbols":["A","B"]}}')
+ pre_schema = Parse(pre)
+ canonical_form = ToParsingCanonicalForm(pre_schema)
+ self.assertEqual(canonical_form, post)
+
+ def testCanonicalization24(self):
+ pre='["string", "null", "long"]'
+ post='["string","null","long"]'
+ pre_schema = Parse(pre)
+ canonical_form = ToParsingCanonicalForm(pre_schema)
+ self.assertEqual(canonical_form, post)
+
+ def testCanonicalization25(self):
+ pre="""
+ {
+ "type": "record",
+ "name": "Test",
+ "fields": [{"name": "f", "type": "long"}]
+ }
+ """
+ post='{"name":"Test","type":"record","fields":[{"name":"f","type":"long"}]}'
+ pre_schema = Parse(pre)
+ canonical_form = ToParsingCanonicalForm(pre_schema)
+ self.assertEqual(canonical_form, post)
+
+ def testCanonicalization26(self):
+ pre="""
+ {
+ "type": "error",
+ "name": "Test",
+ "fields": [{"name": "f", "type": "long"}]
+ }
+ """
+ post='{"name":"Test","type":"record","fields":[{"name":"f","type":"long"}]}'
+ pre_schema = Parse(pre)
+ canonical_form = ToParsingCanonicalForm(pre_schema)
+ self.assertEqual(canonical_form, post)
+
+ def testCanonicalization27(self):
+ pre="""
+ {
+ "type": "record",
+ "name": "Node",
+ "fields": [
+ {"name": "label", "type": "string"},
+ {"name": "children", "type": {"type": "array", "items": "Node"}}
+ ]
+ }
+ """
+ post=('{"name":"Node","type":"record","fields":[{"na'
+ 'me":"label","type":"string"},{"name":"childre'
+ 'n","type":{"type":"array","items":"Node"}}]}')
+ pre_schema = Parse(pre)
+ canonical_form = ToParsingCanonicalForm(pre_schema)
+ self.assertEqual(canonical_form, post)
+
+ def testCanonicalization28(self):
+ pre="""
+ {
+ "type": "record",
+ "name": "Lisp",
+ "fields": [{
+ "name": "value",
+ "type": [
+ "null",
+ "string",
+ {
+ "type": "record",
+ "name": "Cons",
+ "fields": [{"name": "car", "type": "Lisp"},
+ {"name": "cdr", "type": "Lisp"}]
+ }
+ ]
+ }]
+ }
+ """
+ post=('{"name":"Lisp","type":"record","fields":[{"name":"value","type'
+ '":["null","string",{"name":"Cons","type":"record","fields":[{"'
+ 'name":"car","type":"Lisp"},{"name":"cdr","type":"Lisp"}]}]}]}')
+ pre_schema = Parse(pre)
+ canonical_form = ToParsingCanonicalForm(pre_schema)
+ self.assertEqual(canonical_form, post)
+
+ def testCanonicalization29(self):
+ pre="""
+ {
+ "type": "record",
+ "name": "HandshakeRequest",
+ "namespace": "org.apache.avro.ipc",
+ "fields": [
+ {
+ "name": "clientHash",
+ "type": {"type": "fixed", "name": "MD5", "size": 16}
+ },
+ {"name": "clientProtocol", "type": ["null", "string"]},
+ {"name": "serverHash", "type": "MD5"},
+ {
+ "name": "meta",
+ "type": ["null", {"type": "map", "values": "bytes"}]
+ }
+ ]
+ }
+ """
+ post=('{"name":"org.apache.avro.ipc.HandshakeRequest","type":"r'
+ 'ecord","fields":[{"name":"clientHash","type":{"name":"or'
+ 'g.apache.avro.ipc.MD5","type":"fixed","size":16}},{"name'
+ '":"clientProtocol","type":["null","string"]},{"name":"se'
+ 'rverHash","type":"org.apache.avro.ipc.MD5"},{"name":"met'
+ 'a","type":["null",{"type":"map","values":"bytes"}]}]}')
+ pre_schema = Parse(pre)
+ canonical_form = ToParsingCanonicalForm(pre_schema)
+ self.assertEqual(canonical_form, post)
+
+ def testCanonicalization30(self):
+ pre="""
+ {
+ "type": "record",
+ "name": "HandshakeResponse",
+ "namespace": "org.apache.avro.ipc",
+ "fields": [
+ {
+ "name": "match",
+ "type": {
+ "type": "enum",
+ "name": "HandshakeMatch",
+ "symbols": ["BOTH", "CLIENT", "NONE"]
+ }
+ },
+ {"name": "serverProtocol", "type": ["null", "string"]},
+ {
+ "name": "serverHash",
+ "type": ["null", {"name": "MD5", "size": 16, "type": "fixed"}]
+ },
+ {
+ "name": "meta",
+ "type": ["null", {"type": "map", "values": "bytes"}]}]
+ }
+ """
+ post=('{"name":"org.apache.avro.ipc.HandshakeResponse","type":"rec'
+ 'ord","fields":[{"name":"match","type":{"name":"org.apache.a'
+ 'vro.ipc.HandshakeMatch","type":"enum","symbols":["BOTH","CL'
+ 'IENT","NONE"]}},{"name":"serverProtocol","type":["null","st'
+ 'ring"]},{"name":"serverHash","type":["null",{"name":"org.ap'
+ 'ache.avro.ipc.MD5","type":"fixed","size":16}]},{"name":"met'
+ 'a","type":["null",{"type":"map","values":"bytes"}]}]}')
+ pre_schema = Parse(pre)
+ canonical_form = ToParsingCanonicalForm(pre_schema)
+ self.assertEqual(canonical_form, post)
+
+ def testCanonicalization31(self):
+ pre="""
+ {
+ "type": "record",
+ "name": "Interop",
+ "namespace": "org.apache.avro",
+ "fields": [
+ {"name": "intField", "type": "int"},
+ {"name": "longField", "type": "long"},
+ {"name": "stringField", "type": "string"},
+ {"name": "boolField", "type": "boolean"},
+ {"name": "floatField", "type": "float"},
+ {"name": "doubleField", "type": "double"},
+ {"name": "bytesField", "type": "bytes"},
+ {"name": "nullField", "type": "null"},
+ {"name": "arrayField", "type": {"type": "array", "items": "double"}},
+ {
+ "name": "mapField",
+ "type": {
+ "type": "map",
+ "values": {"name": "Foo",
+ "type": "record",
+ "fields": [{"name": "label", "type": "string"}]}
+ }
+ },
+ {
+ "name": "unionField",
+ "type": ["boolean", "double", {"type": "array", "items": "bytes"}]
+ },
+ {
+ "name": "enumField",
+ "type": {"type": "enum", "name": "Kind", "symbols": ["A", "B", "C"]}
+ },
+ {
+ "name": "fixedField",
+ "type": {"type": "fixed", "name": "MD5", "size": 16}
+ },
+ {
+ "name": "recordField",
+ "type": {"type": "record",
+ "name": "Node",
+ "fields": [{"name": "label", "type": "string"},
+ {"name": "children",
+ "type": {"type": "array",
+ "items": "Node"}}]}
+ }
+ ]
+ }
+ """
+ post=('{"name":"org.apache.avro.Interop","type":"record","fields":[{"na'
+ 'me":"intField","type":"int"},{"name":"longField","type":"long"},'
+ '{"name":"stringField","type":"string"},{"name":"boolField","type'
+ '":"boolean"},{"name":"floatField","type":"float"},{"name":"doubl'
+ 'eField","type":"double"},{"name":"bytesField","type":"bytes"},{"'
+ 'name":"nullField","type":"null"},{"name":"arrayField","type":{"t'
+ 'ype":"array","items":"double"}},{"name":"mapField","type":{"type'
+ '":"map","values":{"name":"org.apache.avro.Foo","type":"record","'
+ 'fields":[{"name":"label","type":"string"}]}}},{"name":"unionFiel'
+ 'd","type":["boolean","double",{"type":"array","items":"bytes"}]}'
+ ',{"name":"enumField","type":{"name":"org.apache.avro.Kind","type'
+ '":"enum","symbols":["A","B","C"]}},{"name":"fixedField","type":{'
+ '"name":"org.apache.avro.MD5","type":"fixed","size":16}},{"name":'
+ '"recordField","type":{"name":"org.apache.avro.Node","type":"reco'
+ 'rd","fields":[{"name":"label","type":"string"},{"name":"children'
+ '","type":{"type":"array","items":"org.apache.avro.Node"}}]}}]}')
+ pre_schema = Parse(pre)
+ canonical_form = ToParsingCanonicalForm(pre_schema)
+ self.assertEqual(canonical_form, post)
+
+ def testCanonicalization32(self):
+ pre="""
+ {
+ "type": "record",
+ "name": "ipAddr",
+ "fields": [{
+ "name": "addr",
+ "type": [
+ {"name": "IPv6", "type": "fixed", "size": 16},
+ {"name": "IPv4", "type": "fixed", "size": 4}
+ ]
+ }]
+ }
+ """
+ post=('{"name":"ipAddr","type":"record","fields":[{"name"'
+ ':"addr","type":[{"name":"IPv6","type":"fixed","siz'
+ 'e":16},{"name":"IPv4","type":"fixed","size":4}]}]}')
+ pre_schema = Parse(pre)
+ canonical_form = ToParsingCanonicalForm(pre_schema)
+ self.assertEqual(canonical_form, post)
+
+ def testCanonicalization33(self):
+ pre="""
+ {
+ "type": "record",
+ "name": "TestDoc",
+ "doc": "Doc string",
+ "fields": [{"name": "name", "type": "string", "doc": "Doc String"}]
+ }
+ """
+ post=('{"name":"TestDoc","type":"record","fiel'
+ 'ds":[{"name":"name","type":"string"}]}')
+ pre_schema = Parse(pre)
+ canonical_form = ToParsingCanonicalForm(pre_schema)
+ self.assertEqual(canonical_form, post)
+
+ def testCanonicalization34(self):
+ pre="""
+ {"type": "enum", "name": "Test", "symbols": ["A", "B"], "doc": "Doc String"}
+ """
+ post='{"name":"Test","type":"enum","symbols":["A","B"]}'
+ pre_schema = Parse(pre)
+ canonical_form = ToParsingCanonicalForm(pre_schema)
+ self.assertEqual(canonical_form, post)
+
+
+class TestSchemaFingerprintAlgorithms(unittest.TestCase):
+
+ def testCrc64AvroIsAvailable(self):
+ self.assertIn('CRC-64-AVRO', FingerprintAlgorithmNames())
+
+ def testMd5IsAvailable(self):
+ self.assertIn('md5', FingerprintAlgorithmNames())
+
+ def testSha256IsAvailable(self):
+ self.assertIn('sha256', FingerprintAlgorithmNames())
+
+ def testMd5JavaNameIsAvailable(self):
+ self.assertIn('MD5', FingerprintAlgorithmNames())
+
+ def testSha256JavaNameIsAvailable(self):
+ self.assertIn('SHA-256', FingerprintAlgorithmNames())
+
+class TestSchemaFingerprinting(unittest.TestCase):
+
+ def testUnsupportedFingerprintAlgorithmRaisesValueError(self):
+ schema='"int"'
+ algorithm='UNKNOWN'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ with self.assertRaises(ValueError) as context:
+ Fingerprint(normal_form, algorithm)
+ self.assertIn(algorithm, str(context.exception))
+
+ def testFingerprint1(self):
+ schema='"int"'
+ algorithm='CRC-64-AVRO'
+ expected_hex_fingerprint='8f5c393f1ad57572'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint2(self):
+ schema='"int"'
+ algorithm='md5'
+ expected_hex_fingerprint='ef524ea1b91e73173d938ade36c1db32'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint3(self):
+ schema='"int"'
+ algorithm='sha256'
+ expected_hex_fingerprint='3f2b87a9fe7cc9b13835598c3981cd45e3e355309e5090aa0933d7becb6fba45'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint4(self):
+ schema='{"type": "int"}'
+ algorithm='CRC-64-AVRO'
+ expected_hex_fingerprint='8f5c393f1ad57572'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint5(self):
+ schema='{"type": "int"}'
+ algorithm='md5'
+ expected_hex_fingerprint='ef524ea1b91e73173d938ade36c1db32'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint6(self):
+ schema='{"type": "int"}'
+ algorithm='sha256'
+ expected_hex_fingerprint='3f2b87a9fe7cc9b13835598c3981cd45e3e355309e5090aa0933d7becb6fba45'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint7(self):
+ schema='"float"'
+ algorithm='CRC-64-AVRO'
+ expected_hex_fingerprint='90d7a83ecb027c4d'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint8(self):
+ schema='"float"'
+ algorithm='md5'
+ expected_hex_fingerprint='50a6b9db85da367a6d2df400a41758a6'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint9(self):
+ schema='"float"'
+ algorithm='sha256'
+ expected_hex_fingerprint='1e71f9ec051d663f56b0d8e1fc84d71aa56ccfe9fa93aa20d10547a7abeb5cc0'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint10(self):
+ schema='{"type": "float"}'
+ algorithm='CRC-64-AVRO'
+ expected_hex_fingerprint='90d7a83ecb027c4d'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint11(self):
+ schema='{"type": "float"}'
+ algorithm='md5'
+ expected_hex_fingerprint='50a6b9db85da367a6d2df400a41758a6'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint12(self):
+ schema='{"type": "float"}'
+ algorithm='sha256'
+ expected_hex_fingerprint='1e71f9ec051d663f56b0d8e1fc84d71aa56ccfe9fa93aa20d10547a7abeb5cc0'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint13(self):
+ schema='"long"'
+ algorithm='CRC-64-AVRO'
+ expected_hex_fingerprint='b71df49344e154d0'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint14(self):
+ schema='"long"'
+ algorithm='md5'
+ expected_hex_fingerprint='e1dd9a1ef98b451b53690370b393966b'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint15(self):
+ schema='"long"'
+ algorithm='sha256'
+ expected_hex_fingerprint='c32c497df6730c97fa07362aa5023f37d49a027ec452360778114cf427965add'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint16(self):
+ schema='{"type": "long"}'
+ algorithm='CRC-64-AVRO'
+ expected_hex_fingerprint='b71df49344e154d0'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint17(self):
+ schema='{"type": "long"}'
+ algorithm='md5'
+ expected_hex_fingerprint='e1dd9a1ef98b451b53690370b393966b'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint18(self):
+ schema='{"type": "long"}'
+ algorithm='sha256'
+ expected_hex_fingerprint='c32c497df6730c97fa07362aa5023f37d49a027ec452360778114cf427965add'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint19(self):
+ schema='"double"'
+ algorithm='CRC-64-AVRO'
+ expected_hex_fingerprint='7e95ab32c035758e'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint20(self):
+ schema='"double"'
+ algorithm='md5'
+ expected_hex_fingerprint='bfc71a62f38b99d6a93690deeb4b3af6'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint21(self):
+ schema='"double"'
+ algorithm='sha256'
+ expected_hex_fingerprint='730a9a8c611681d7eef442e03c16c70d13bca3eb8b977bb403eaff52176af254'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint22(self):
+ schema='{"type": "double"}'
+ algorithm='CRC-64-AVRO'
+ expected_hex_fingerprint='7e95ab32c035758e'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint23(self):
+ schema='{"type": "double"}'
+ algorithm='md5'
+ expected_hex_fingerprint='bfc71a62f38b99d6a93690deeb4b3af6'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint24(self):
+ schema='{"type": "double"}'
+ algorithm='sha256'
+ expected_hex_fingerprint='730a9a8c611681d7eef442e03c16c70d13bca3eb8b977bb403eaff52176af254'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint25(self):
+ schema='"bytes"'
+ algorithm='CRC-64-AVRO'
+ expected_hex_fingerprint='651920c3da16c04f'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint26(self):
+ schema='"bytes"'
+ algorithm='md5'
+ expected_hex_fingerprint='b462f06cb909be57c85008867784cde6'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint27(self):
+ schema='"bytes"'
+ algorithm='sha256'
+ expected_hex_fingerprint='9ae507a9dd39ee5b7c7e285da2c0846521c8ae8d80feeae5504e0c981d53f5fa'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint28(self):
+ schema='{"type": "bytes"}'
+ algorithm='CRC-64-AVRO'
+ expected_hex_fingerprint='651920c3da16c04f'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint29(self):
+ schema='{"type": "bytes"}'
+ algorithm='md5'
+ expected_hex_fingerprint='b462f06cb909be57c85008867784cde6'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint30(self):
+ schema='{"type": "bytes"}'
+ algorithm='sha256'
+ expected_hex_fingerprint='9ae507a9dd39ee5b7c7e285da2c0846521c8ae8d80feeae5504e0c981d53f5fa'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint31(self):
+ schema='"string"'
+ algorithm='CRC-64-AVRO'
+ expected_hex_fingerprint='c70345637248018f'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint32(self):
+ schema='"string"'
+ algorithm='md5'
+ expected_hex_fingerprint='095d71cf12556b9d5e330ad575b3df5d'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint33(self):
+ schema='"string"'
+ algorithm='sha256'
+ expected_hex_fingerprint='e9e5c1c9e4f6277339d1bcde0733a59bd42f8731f449da6dc13010a916930d48'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint34(self):
+ schema='{"type": "string"}'
+ algorithm='CRC-64-AVRO'
+ expected_hex_fingerprint='c70345637248018f'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint35(self):
+ schema='{"type": "string"}'
+ algorithm='md5'
+ expected_hex_fingerprint='095d71cf12556b9d5e330ad575b3df5d'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint36(self):
+ schema='{"type": "string"}'
+ algorithm='sha256'
+ expected_hex_fingerprint='e9e5c1c9e4f6277339d1bcde0733a59bd42f8731f449da6dc13010a916930d48'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint37(self):
+ schema='"boolean"'
+ algorithm='CRC-64-AVRO'
+ expected_hex_fingerprint='64f7d4a478fc429f'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint38(self):
+ schema='"boolean"'
+ algorithm='md5'
+ expected_hex_fingerprint='01f692b30d4a1c8a3e600b1440637f8f'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint39(self):
+ schema='"boolean"'
+ algorithm='sha256'
+ expected_hex_fingerprint='a5b031ab62bc416d720c0410d802ea46b910c4fbe85c50a946ccc658b74e677e'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint40(self):
+ schema='{"type": "boolean"}'
+ algorithm='CRC-64-AVRO'
+ expected_hex_fingerprint='64f7d4a478fc429f'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint41(self):
+ schema='{"type": "boolean"}'
+ algorithm='md5'
+ expected_hex_fingerprint='01f692b30d4a1c8a3e600b1440637f8f'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint42(self):
+ schema='{"type": "boolean"}'
+ algorithm='sha256'
+ expected_hex_fingerprint='a5b031ab62bc416d720c0410d802ea46b910c4fbe85c50a946ccc658b74e677e'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint43(self):
+ schema='"null"'
+ algorithm='CRC-64-AVRO'
+ expected_hex_fingerprint='8a8f25cce724dd63'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint44(self):
+ schema='"null"'
+ algorithm='md5'
+ expected_hex_fingerprint='9b41ef67651c18488a8b08bb67c75699'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint45(self):
+ schema='"null"'
+ algorithm='sha256'
+ expected_hex_fingerprint='f072cbec3bf8841871d4284230c5e983dc211a56837aed862487148f947d1a1f'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint46(self):
+ schema='{"type": "null"}'
+ algorithm='CRC-64-AVRO'
+ expected_hex_fingerprint='8a8f25cce724dd63'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint47(self):
+ schema='{"type": "null"}'
+ algorithm='md5'
+ expected_hex_fingerprint='9b41ef67651c18488a8b08bb67c75699'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint48(self):
+ schema='{"type": "null"}'
+ algorithm='sha256'
+ expected_hex_fingerprint='f072cbec3bf8841871d4284230c5e983dc211a56837aed862487148f947d1a1f'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint49(self):
+ schema='{"type": "fixed", "name": "Test", "size": 1}'
+ algorithm='CRC-64-AVRO'
+ expected_hex_fingerprint='6869897b4049355b'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint50(self):
+ schema='{"type": "fixed", "name": "Test", "size": 1}'
+ algorithm='md5'
+ expected_hex_fingerprint='db01bc515fcfcd2d4be82ed385288261'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint51(self):
+ schema='{"type": "fixed", "name": "Test", "size": 1}'
+ algorithm='sha256'
+ expected_hex_fingerprint='f527116a6f44455697e935afc31dc60ad0f95caf35e1d9c9db62edb3ffeb9170'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint52(self):
+ schema="""
+ {
+ "type": "fixed",
+ "name": "MyFixed",
+ "namespace": "org.apache.hadoop.avro",
+ "size": 1
+ }
+ """
+ algorithm='CRC-64-AVRO'
+ expected_hex_fingerprint='fadbd138e85bdf45'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint53(self):
+ schema="""
+ {
+ "type": "fixed",
+ "name": "MyFixed",
+ "namespace": "org.apache.hadoop.avro",
+ "size": 1
+ }
+ """
+ algorithm='md5'
+ expected_hex_fingerprint='d74b3726484422711c465d49e857b1ba'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint54(self):
+ schema="""
+ {
+ "type": "fixed",
+ "name": "MyFixed",
+ "namespace": "org.apache.hadoop.avro",
+ "size": 1
+ }
+ """
+ algorithm='sha256'
+ expected_hex_fingerprint='28e493a44771cecc5deca4bd938cdc3d5a24cfe1f3760bc938fa1057df6334fc'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint55(self):
+ schema='{"type": "enum", "name": "Test", "symbols": ["A", "B"]}'
+ algorithm='CRC-64-AVRO'
+ expected_hex_fingerprint='03a2f2c2e27f7a16'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint56(self):
+ schema='{"type": "enum", "name": "Test", "symbols": ["A", "B"]}'
+ algorithm='md5'
+ expected_hex_fingerprint='d883f2a9b16ed085fcc5e4ca6c8f6ed1'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint57(self):
+ schema='{"type": "enum", "name": "Test", "symbols": ["A", "B"]}'
+ algorithm='sha256'
+ expected_hex_fingerprint='9b51286144f87ce5aebdc61ca834379effa5a41ce6ac0938630ff246297caca8'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint58(self):
+ schema='{"type": "array", "items": "long"}'
+ algorithm='CRC-64-AVRO'
+ expected_hex_fingerprint='715e2ea28bc91654'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint59(self):
+ schema='{"type": "array", "items": "long"}'
+ algorithm='md5'
+ expected_hex_fingerprint='c1c387e8d6a58f0df749b698991b1f43'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint60(self):
+ schema='{"type": "array", "items": "long"}'
+ algorithm='sha256'
+ expected_hex_fingerprint='f78e954167feb23dcb1ce01e8463cebf3408e0a4259e16f24bd38f6d0f1d578b'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint61(self):
+ schema="""
+ {
+ "type": "array",
+ "items": {"type": "enum", "name": "Test", "symbols": ["A", "B"]}
+ }
+ """
+ algorithm='CRC-64-AVRO'
+ expected_hex_fingerprint='10d9ade1fa3a0387'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint62(self):
+ schema="""
+ {
+ "type": "array",
+ "items": {"type": "enum", "name": "Test", "symbols": ["A", "B"]}
+ }
+ """
+ algorithm='md5'
+ expected_hex_fingerprint='cfc7b861c7cfef082a6ef082948893fa'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint63(self):
+ schema="""
+ {
+ "type": "array",
+ "items": {"type": "enum", "name": "Test", "symbols": ["A", "B"]}
+ }
+ """
+ algorithm='sha256'
+ expected_hex_fingerprint='0d8edd49d7f7e9553668f133577bc99f842852b55d9f84f1f7511e4961aa685c'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint64(self):
+ schema='{"type": "map", "values": "long"}'
+ algorithm='CRC-64-AVRO'
+ expected_hex_fingerprint='6f74f4e409b1334e'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint65(self):
+ schema='{"type": "map", "values": "long"}'
+ algorithm='md5'
+ expected_hex_fingerprint='32b3f1a3177a0e73017920f00448b56e'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint66(self):
+ schema='{"type": "map", "values": "long"}'
+ algorithm='sha256'
+ expected_hex_fingerprint='b8fad07d458971a07692206b8a7cf626c86c62fe6bcff7c1b11bc7295de34853'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint67(self):
+ schema="""
+ {
+ "type": "map",
+ "values": {"type": "enum", "name": "Test", "symbols": ["A", "B"]}
+ }
+ """
+ algorithm='CRC-64-AVRO'
+ expected_hex_fingerprint='df2ab0626f6b812d'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint68(self):
+ schema="""
+ {
+ "type": "map",
+ "values": {"type": "enum", "name": "Test", "symbols": ["A", "B"]}
+ }
+ """
+ algorithm='md5'
+ expected_hex_fingerprint='c588da6ba99701c41e73fd30d23f994e'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint69(self):
+ schema="""
+ {
+ "type": "map",
+ "values": {"type": "enum", "name": "Test", "symbols": ["A", "B"]}
+ }
+ """
+ algorithm='sha256'
+ expected_hex_fingerprint='3886747ed1669a8af476b549e97b34222afb2fed5f18bb27c6f367ea0351a576'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint70(self):
+ schema='["string", "null", "long"]'
+ algorithm='CRC-64-AVRO'
+ expected_hex_fingerprint='65a5be410d687566'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint71(self):
+ schema='["string", "null", "long"]'
+ algorithm='md5'
+ expected_hex_fingerprint='b11cf95f0a55dd55f9ee515a37bf937a'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint72(self):
+ schema='["string", "null", "long"]'
+ algorithm='sha256'
+ expected_hex_fingerprint='ed8d254116441bb35e237ad0563cf5432b8c975334bd222c1ee84609435d95bb'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint73(self):
+ schema="""
+ {
+ "type": "record",
+ "name": "Test",
+ "fields": [{"name": "f", "type": "long"}]
+ }
+ """
+ algorithm='CRC-64-AVRO'
+ expected_hex_fingerprint='ed94e5f5e6eb588e'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint74(self):
+ schema="""
+ {
+ "type": "record",
+ "name": "Test",
+ "fields": [{"name": "f", "type": "long"}]
+ }
+ """
+ algorithm='md5'
+ expected_hex_fingerprint='69531a03db788afe353244cd049b1e6d'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint75(self):
+ schema="""
+ {
+ "type": "record",
+ "name": "Test",
+ "fields": [{"name": "f", "type": "long"}]
+ }
+ """
+ algorithm='sha256'
+ expected_hex_fingerprint='9670f15a8f96d23e92830d00b8bd57275e02e3e173ffef7c253c170b6beabeb8'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint76(self):
+ schema="""
+ {
+ "type": "error",
+ "name": "Test",
+ "fields": [{"name": "f", "type": "long"}]
+ }
+ """
+ algorithm='CRC-64-AVRO'
+ expected_hex_fingerprint='ed94e5f5e6eb588e'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint77(self):
+ schema="""
+ {
+ "type": "error",
+ "name": "Test",
+ "fields": [{"name": "f", "type": "long"}]
+ }
+ """
+ algorithm='md5'
+ expected_hex_fingerprint='69531a03db788afe353244cd049b1e6d'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint78(self):
+ schema="""
+ {
+ "type": "error",
+ "name": "Test",
+ "fields": [{"name": "f", "type": "long"}]
+ }
+ """
+ algorithm='sha256'
+ expected_hex_fingerprint='9670f15a8f96d23e92830d00b8bd57275e02e3e173ffef7c253c170b6beabeb8'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint79(self):
+ schema="""
+ {
+ "type": "record",
+ "name": "Node",
+ "fields": [
+ {"name": "label", "type": "string"},
+ {"name": "children", "type": {"type": "array", "items": "Node"}}
+ ]
+ }
+ """
+ algorithm='CRC-64-AVRO'
+ expected_hex_fingerprint='52cba544c3e756b7'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint80(self):
+ schema="""
+ {
+ "type": "record",
+ "name": "Node",
+ "fields": [
+ {"name": "label", "type": "string"},
+ {"name": "children", "type": {"type": "array", "items": "Node"}}
+ ]
+ }
+ """
+ algorithm='md5'
+ expected_hex_fingerprint='99625b0cc02050363e89ef66b0f406c9'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint81(self):
+ schema="""
+ {
+ "type": "record",
+ "name": "Node",
+ "fields": [
+ {"name": "label", "type": "string"},
+ {"name": "children", "type": {"type": "array", "items": "Node"}}
+ ]
+ }
+ """
+ algorithm='sha256'
+ expected_hex_fingerprint='65d80dc8c95c98a9671d92cf0415edfabfee2cb058df2138606656cd6ae4dc59'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint82(self):
+ schema="""
+ {
+ "type": "record",
+ "name": "Lisp",
+ "fields": [{
+ "name": "value",
+ "type": [
+ "null",
+ "string",
+ {
+ "type": "record",
+ "name": "Cons",
+ "fields": [{"name": "car", "type": "Lisp"},
+ {"name": "cdr", "type": "Lisp"}]
+ }
+ ]
+ }]
+ }
+ """
+ algorithm='CRC-64-AVRO'
+ expected_hex_fingerprint='68d91a23eda0b306'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint83(self):
+ schema="""
+ {
+ "type": "record",
+ "name": "Lisp",
+ "fields": [{
+ "name": "value",
+ "type": [
+ "null",
+ "string",
+ {
+ "type": "record",
+ "name": "Cons",
+ "fields": [{"name": "car", "type": "Lisp"},
+ {"name": "cdr", "type": "Lisp"}]
+ }
+ ]
+ }]
+ }
+ """
+ algorithm='md5'
+ expected_hex_fingerprint='9e1d0d15b52789fcb8e3a88b53059d5f'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint84(self):
+ schema="""
+ {
+ "type": "record",
+ "name": "Lisp",
+ "fields": [{
+ "name": "value",
+ "type": [
+ "null",
+ "string",
+ {
+ "type": "record",
+ "name": "Cons",
+ "fields": [{"name": "car", "type": "Lisp"},
+ {"name": "cdr", "type": "Lisp"}]
+ }
+ ]
+ }]
+ }
+ """
+ algorithm='sha256'
+ expected_hex_fingerprint='e5ce4f4a15ce19fa1047cfe16a3b0e13a755db40f00f23284fdd376fc1c7dd21'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint85(self):
+ schema="""
+ {
+ "type": "record",
+ "name": "HandshakeRequest",
+ "namespace": "org.apache.avro.ipc",
+ "fields": [
+ {
+ "name": "clientHash",
+ "type": {"type": "fixed", "name": "MD5", "size": 16}
+ },
+ {"name": "clientProtocol", "type": ["null", "string"]},
+ {"name": "serverHash", "type": "MD5"},
+ {
+ "name": "meta",
+ "type": ["null", {"type": "map", "values": "bytes"}]
+ }
+ ]
+ }
+ """
+ algorithm='CRC-64-AVRO'
+ expected_hex_fingerprint='b96ad79e5a7c5757'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint86(self):
+ schema="""
+ {
+ "type": "record",
+ "name": "HandshakeRequest",
+ "namespace": "org.apache.avro.ipc",
+ "fields": [
+ {
+ "name": "clientHash",
+ "type": {"type": "fixed", "name": "MD5", "size": 16}
+ },
+ {"name": "clientProtocol", "type": ["null", "string"]},
+ {"name": "serverHash", "type": "MD5"},
+ {
+ "name": "meta",
+ "type": ["null", {"type": "map", "values": "bytes"}]
+ }
+ ]
+ }
+ """
+ algorithm='md5'
+ expected_hex_fingerprint='4c822af2e17eecd92422827eede97f5b'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint87(self):
+ schema="""
+ {
+ "type": "record",
+ "name": "HandshakeRequest",
+ "namespace": "org.apache.avro.ipc",
+ "fields": [
+ {
+ "name": "clientHash",
+ "type": {"type": "fixed", "name": "MD5", "size": 16}
+ },
+ {"name": "clientProtocol", "type": ["null", "string"]},
+ {"name": "serverHash", "type": "MD5"},
+ {
+ "name": "meta",
+ "type": ["null", {"type": "map", "values": "bytes"}]
+ }
+ ]
+ }
+ """
+ algorithm='sha256'
+ expected_hex_fingerprint='2b2f7a9b22991fe0df9134cb6b5ff7355343e797aaea337e0150e20f3a35800e'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint88(self):
+ schema="""
+ {
+ "type": "record",
+ "name": "HandshakeResponse",
+ "namespace": "org.apache.avro.ipc",
+ "fields": [
+ {
+ "name": "match",
+ "type": {
+ "type": "enum",
+ "name": "HandshakeMatch",
+ "symbols": ["BOTH", "CLIENT", "NONE"]
+ }
+ },
+ {"name": "serverProtocol", "type": ["null", "string"]},
+ {
+ "name": "serverHash",
+ "type": ["null", {"name": "MD5", "size": 16, "type": "fixed"}]
+ },
+ {
+ "name": "meta",
+ "type": ["null", {"type": "map", "values": "bytes"}]}]
+ }
+ """
+ algorithm='CRC-64-AVRO'
+ expected_hex_fingerprint='00feee01de4ea50e'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint89(self):
+ schema="""
+ {
+ "type": "record",
+ "name": "HandshakeResponse",
+ "namespace": "org.apache.avro.ipc",
+ "fields": [
+ {
+ "name": "match",
+ "type": {
+ "type": "enum",
+ "name": "HandshakeMatch",
+ "symbols": ["BOTH", "CLIENT", "NONE"]
+ }
+ },
+ {"name": "serverProtocol", "type": ["null", "string"]},
+ {
+ "name": "serverHash",
+ "type": ["null", {"name": "MD5", "size": 16, "type": "fixed"}]
+ },
+ {
+ "name": "meta",
+ "type": ["null", {"type": "map", "values": "bytes"}]}]
+ }
+ """
+ algorithm='md5'
+ expected_hex_fingerprint='afe529d01132daab7f4e2a6663e7a2f5'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint90(self):
+ schema="""
+ {
+ "type": "record",
+ "name": "HandshakeResponse",
+ "namespace": "org.apache.avro.ipc",
+ "fields": [
+ {
+ "name": "match",
+ "type": {
+ "type": "enum",
+ "name": "HandshakeMatch",
+ "symbols": ["BOTH", "CLIENT", "NONE"]
+ }
+ },
+ {"name": "serverProtocol", "type": ["null", "string"]},
+ {
+ "name": "serverHash",
+ "type": ["null", {"name": "MD5", "size": 16, "type": "fixed"}]
+ },
+ {
+ "name": "meta",
+ "type": ["null", {"type": "map", "values": "bytes"}]}]
+ }
+ """
+ algorithm='sha256'
+ expected_hex_fingerprint='a303cbbfe13958f880605d70c521a4b7be34d9265ac5a848f25916a67b11d889'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint91(self):
+ schema="""
+ {
+ "type": "record",
+ "name": "Interop",
+ "namespace": "org.apache.avro",
+ "fields": [
+ {"name": "intField", "type": "int"},
+ {"name": "longField", "type": "long"},
+ {"name": "stringField", "type": "string"},
+ {"name": "boolField", "type": "boolean"},
+ {"name": "floatField", "type": "float"},
+ {"name": "doubleField", "type": "double"},
+ {"name": "bytesField", "type": "bytes"},
+ {"name": "nullField", "type": "null"},
+ {"name": "arrayField", "type": {"type": "array", "items": "double"}},
+ {
+ "name": "mapField",
+ "type": {
+ "type": "map",
+ "values": {"name": "Foo",
+ "type": "record",
+ "fields": [{"name": "label", "type": "string"}]}
+ }
+ },
+ {
+ "name": "unionField",
+ "type": ["boolean", "double", {"type": "array", "items": "bytes"}]
+ },
+ {
+ "name": "enumField",
+ "type": {"type": "enum", "name": "Kind", "symbols": ["A", "B", "C"]}
+ },
+ {
+ "name": "fixedField",
+ "type": {"type": "fixed", "name": "MD5", "size": 16}
+ },
+ {
+ "name": "recordField",
+ "type": {"type": "record",
+ "name": "Node",
+ "fields": [{"name": "label", "type": "string"},
+ {"name": "children",
+ "type": {"type": "array",
+ "items": "Node"}}]}
+ }
+ ]
+ }
+ """
+ algorithm='CRC-64-AVRO'
+ expected_hex_fingerprint='e82c0a93a6a0b5a4'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint92(self):
+ schema="""
+ {
+ "type": "record",
+ "name": "Interop",
+ "namespace": "org.apache.avro",
+ "fields": [
+ {"name": "intField", "type": "int"},
+ {"name": "longField", "type": "long"},
+ {"name": "stringField", "type": "string"},
+ {"name": "boolField", "type": "boolean"},
+ {"name": "floatField", "type": "float"},
+ {"name": "doubleField", "type": "double"},
+ {"name": "bytesField", "type": "bytes"},
+ {"name": "nullField", "type": "null"},
+ {"name": "arrayField", "type": {"type": "array", "items": "double"}},
+ {
+ "name": "mapField",
+ "type": {
+ "type": "map",
+ "values": {"name": "Foo",
+ "type": "record",
+ "fields": [{"name": "label", "type": "string"}]}
+ }
+ },
+ {
+ "name": "unionField",
+ "type": ["boolean", "double", {"type": "array", "items": "bytes"}]
+ },
+ {
+ "name": "enumField",
+ "type": {"type": "enum", "name": "Kind", "symbols": ["A", "B", "C"]}
+ },
+ {
+ "name": "fixedField",
+ "type": {"type": "fixed", "name": "MD5", "size": 16}
+ },
+ {
+ "name": "recordField",
+ "type": {"type": "record",
+ "name": "Node",
+ "fields": [{"name": "label", "type": "string"},
+ {"name": "children",
+ "type": {"type": "array",
+ "items": "Node"}}]}
+ }
+ ]
+ }
+ """
+ algorithm='md5'
+ expected_hex_fingerprint='994fea1a1be7ff8603cbe40c3bc7e4ca'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint93(self):
+ schema="""
+ {
+ "type": "record",
+ "name": "Interop",
+ "namespace": "org.apache.avro",
+ "fields": [
+ {"name": "intField", "type": "int"},
+ {"name": "longField", "type": "long"},
+ {"name": "stringField", "type": "string"},
+ {"name": "boolField", "type": "boolean"},
+ {"name": "floatField", "type": "float"},
+ {"name": "doubleField", "type": "double"},
+ {"name": "bytesField", "type": "bytes"},
+ {"name": "nullField", "type": "null"},
+ {"name": "arrayField", "type": {"type": "array", "items": "double"}},
+ {
+ "name": "mapField",
+ "type": {
+ "type": "map",
+ "values": {"name": "Foo",
+ "type": "record",
+ "fields": [{"name": "label", "type": "string"}]}
+ }
+ },
+ {
+ "name": "unionField",
+ "type": ["boolean", "double", {"type": "array", "items": "bytes"}]
+ },
+ {
+ "name": "enumField",
+ "type": {"type": "enum", "name": "Kind", "symbols": ["A", "B", "C"]}
+ },
+ {
+ "name": "fixedField",
+ "type": {"type": "fixed", "name": "MD5", "size": 16}
+ },
+ {
+ "name": "recordField",
+ "type": {"type": "record",
+ "name": "Node",
+ "fields": [{"name": "label", "type": "string"},
+ {"name": "children",
+ "type": {"type": "array",
+ "items": "Node"}}]}
+ }
+ ]
+ }
+ """
+ algorithm='sha256'
+ expected_hex_fingerprint='cccfd6e3f917cf53b0f90c206342e6703b0d905071f724a1c1f85b731c74058d'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint94(self):
+ schema="""
+ {
+ "type": "record",
+ "name": "ipAddr",
+ "fields": [{
+ "name": "addr",
+ "type": [
+ {"name": "IPv6", "type": "fixed", "size": 16},
+ {"name": "IPv4", "type": "fixed", "size": 4}
+ ]
+ }]
+ }
+ """
+ algorithm='CRC-64-AVRO'
+ expected_hex_fingerprint='8d961b4e298a1844'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint95(self):
+ schema="""
+ {
+ "type": "record",
+ "name": "ipAddr",
+ "fields": [{
+ "name": "addr",
+ "type": [
+ {"name": "IPv6", "type": "fixed", "size": 16},
+ {"name": "IPv4", "type": "fixed", "size": 4}
+ ]
+ }]
+ }
+ """
+ algorithm='md5'
+ expected_hex_fingerprint='45d85c69b353a99b93d7c4f2fcf0c30d'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint96(self):
+ schema="""
+ {
+ "type": "record",
+ "name": "ipAddr",
+ "fields": [{
+ "name": "addr",
+ "type": [
+ {"name": "IPv6", "type": "fixed", "size": 16},
+ {"name": "IPv4", "type": "fixed", "size": 4}
+ ]
+ }]
+ }
+ """
+ algorithm='sha256'
+ expected_hex_fingerprint='6f6fc8f685a4f07d99734946565d63108806d55a8620febea047cf52cb0ac181'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint97(self):
+ schema="""
+ {
+ "type": "record",
+ "name": "TestDoc",
+ "doc": "Doc string",
+ "fields": [{"name": "name", "type": "string", "doc": "Doc String"}]
+ }
+ """
+ algorithm='CRC-64-AVRO'
+ expected_hex_fingerprint='0e6660f02bcdc109'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint98(self):
+ schema="""
+ {
+ "type": "record",
+ "name": "TestDoc",
+ "doc": "Doc string",
+ "fields": [{"name": "name", "type": "string", "doc": "Doc String"}]
+ }
+ """
+ algorithm='md5'
+ expected_hex_fingerprint='f2da75f5131f5ab80629538287b8beb2'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint99(self):
+ schema="""
+ {
+ "type": "record",
+ "name": "TestDoc",
+ "doc": "Doc string",
+ "fields": [{"name": "name", "type": "string", "doc": "Doc String"}]
+ }
+ """
+ algorithm='sha256'
+ expected_hex_fingerprint='0b3644f7aa5ca2fc4bad93ca2d3609c12aa9dbda9c15e68b34c120beff08e7b9'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint100(self):
+ schema="""
+ {"type": "enum", "name": "Test", "symbols": ["A", "B"], "doc": "Doc String"}
+ """
+ algorithm='CRC-64-AVRO'
+ expected_hex_fingerprint='03a2f2c2e27f7a16'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint101(self):
+ schema="""
+ {"type": "enum", "name": "Test", "symbols": ["A", "B"], "doc": "Doc String"}
+ """
+ algorithm='md5'
+ expected_hex_fingerprint='d883f2a9b16ed085fcc5e4ca6c8f6ed1'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint102(self):
+ schema="""
+ {"type": "enum", "name": "Test", "symbols": ["A", "B"], "doc": "Doc String"}
+ """
+ algorithm='sha256'
+ expected_hex_fingerprint='9b51286144f87ce5aebdc61ca834379effa5a41ce6ac0938630ff246297caca8'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint103(self):
+ schema='{"type": "int"}'
+ algorithm='MD5' # Java compatible name alias
+ expected_hex_fingerprint='ef524ea1b91e73173d938ade36c1db32'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
+
+ def testFingerprint104(self):
+ schema='{"type": "int"}'
+ algorithm='SHA-256' # Java compatible name alias
+ expected_hex_fingerprint='3f2b87a9fe7cc9b13835598c3981cd45e3e355309e5090aa0933d7becb6fba45'
+ pre_schema = Parse(schema)
+ normal_form = ToParsingCanonicalForm(pre_schema)
+ fingerprint = Fingerprint(normal_form, algorithm)
+ actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
+ self.assertEqual(actual_hex_fingerprint, expected_hex_fingerprint)
[avro] 22/23: AVRO-2456: Add interop test for the snappy and zstd
codec (#573)
Posted by rs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rskraba pushed a commit to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/avro.git
commit f6aee38d6cedf43d65251c75226be7c5121c8796
Author: Kengo Seki <se...@apache.org>
AuthorDate: Sat Jul 13 02:55:42 2019 +0900
AVRO-2456: Add interop test for the snappy and zstd codec (#573)
---
lang/java/avro/pom.xml | 32 ++++++++++++++++++++++++++++++--
lang/java/ipc/pom.xml | 8 ++++++++
lang/java/pom.xml | 1 -
lang/ruby/interop/test_interop.rb | 10 +++++++++-
4 files changed, 47 insertions(+), 4 deletions(-)
diff --git a/lang/java/avro/pom.xml b/lang/java/avro/pom.xml
index 5d902b9..4953fce 100644
--- a/lang/java/avro/pom.xml
+++ b/lang/java/avro/pom.xml
@@ -133,6 +133,36 @@
</configuration>
<goals><goal>java</goal></goals>
</execution>
+ <!-- Generate random data for interop tests, using snappy codec -->
+ <execution>
+ <id>interop-generate-snappy-codec</id>
+ <phase>generate-resources</phase>
+ <configuration>
+ <mainClass>org.apache.avro.util.RandomData</mainClass>
+ <arguments>
+ <argument>../../../share/test/schemas/interop.avsc</argument>
+ <argument>../../../build/interop/data/java_snappy.avro</argument>
+ <argument>100</argument>
+ <argument>snappy</argument>
+ </arguments>
+ </configuration>
+ <goals><goal>java</goal></goals>
+ </execution>
+ <!-- Generate random data for interop tests, using zstandard codec -->
+ <execution>
+ <id>interop-generate-zstandard-codec</id>
+ <phase>generate-resources</phase>
+ <configuration>
+ <mainClass>org.apache.avro.util.RandomData</mainClass>
+ <arguments>
+ <argument>../../../share/test/schemas/interop.avsc</argument>
+ <argument>../../../build/interop/data/java_zstandard.avro</argument>
+ <argument>100</argument>
+ <argument>zstandard</argument>
+ </arguments>
+ </configuration>
+ <goals><goal>java</goal></goals>
+ </execution>
</executions>
</plugin>
</plugins>
@@ -156,7 +186,6 @@
<dependency>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
- <scope>provided</scope>
<optional>true</optional>
</dependency>
<dependency>
@@ -169,7 +198,6 @@
<dependency>
<groupId>com.github.luben</groupId>
<artifactId>zstd-jni</artifactId>
- <scope>provided</scope>
<optional>true</optional>
</dependency>
<dependency>
diff --git a/lang/java/ipc/pom.xml b/lang/java/ipc/pom.xml
index 9c68d8f..33fdebc 100644
--- a/lang/java/ipc/pom.xml
+++ b/lang/java/ipc/pom.xml
@@ -149,6 +149,14 @@
<artifactId>javax.annotation-api</artifactId>
<version>1.3.2</version>
</dependency>
+ <dependency>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.github.luben</groupId>
+ <artifactId>zstd-jni</artifactId>
+ </dependency>
</dependencies>
diff --git a/lang/java/pom.xml b/lang/java/pom.xml
index 3bd4c20..3af1890 100644
--- a/lang/java/pom.xml
+++ b/lang/java/pom.xml
@@ -598,7 +598,6 @@
<groupId>com.github.luben</groupId>
<artifactId>zstd-jni</artifactId>
<version>${zstd-jni.version}</version>
- <scope>test</scope>
</dependency>
</dependencies>
</dependencyManagement>
diff --git a/lang/ruby/interop/test_interop.rb b/lang/ruby/interop/test_interop.rb
index 1a4aa74..a63c158 100644
--- a/lang/ruby/interop/test_interop.rb
+++ b/lang/ruby/interop/test_interop.rb
@@ -19,11 +19,19 @@ require 'rubygems'
require 'test/unit'
require 'avro'
+CODECS_TO_VALIDATE = ['deflate'] # The 'null' codec is implicitly included
+
class TestInterop < Test::Unit::TestCase
HERE = File.expand_path(File.dirname(__FILE__))
SHARE = HERE + '/../../../share'
SCHEMAS = SHARE + '/test/schemas'
- Dir[HERE + '/../../../build/interop/data/*'].each do |fn|
+
+ files = Dir[HERE + '/../../../build/interop/data/*.avro'].select do |fn|
+ sep, codec = File.basename(fn, 'avro').rpartition('_')[1, 2]
+ sep.empty? || CODECS_TO_VALIDATE.include?(codec)
+ end
+
+ files.each do |fn|
define_method("test_read_#{File.basename(fn, 'avro')}") do
projection = Avro::Schema.parse(File.read(SCHEMAS+'/interop.avsc'))
[avro] 10/23: AVRO-2576: Fix Premature Import pycodestyle (#664)
Posted by rs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rskraba pushed a commit to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/avro.git
commit a9f05b4f1a6316ffc5add33dbd2edf9012a093f3
Author: Michael A. Smith <mi...@smith-li.com>
AuthorDate: Wed Oct 9 16:32:42 2019 -0400
AVRO-2576: Fix Premature Import pycodestyle (#664)
* AVRO-2445: Remove SimpleJSON License and Rat Excludes
* AVRO-2576: Fix Premature Import pycodestyle
* AVRO-2576: Run pycodestyle from setup for py2
* AVRO-2576: Fix and Ignore Lint Failures
* AVRO-2576: Update Setuptools
* AVRO-2576: Use Local pycodestyle If We Can
* AVRO-2576: Run isort from setuptools
* AVRO-2576: Include isort at setuptools time
* AVRO-2576: Rat Exclude Python Setup Files
---
LICENSE.txt | 25 --------------
lang/py/.gitignore | 3 +-
lang/py/build.sh | 3 +-
lang/py/setup.cfg | 4 +--
lang/py/setup.py | 50 +++++++++++++++++++++++++--
lang/py/src/avro/protocol.py | 1 +
lang/py/src/avro/tether/tether_task_runner.py | 2 +-
lang/py3/.gitignore | 1 +
lang/py3/setup.cfg | 6 +++-
lang/py3/setup.py | 9 ++++-
pom.xml | 44 +++++++++++++----------
share/docker/Dockerfile | 13 +++++--
12 files changed, 104 insertions(+), 57 deletions(-)
diff --git a/LICENSE.txt b/LICENSE.txt
index 4a89170..7e159a6 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -296,31 +296,6 @@ Copyright (C) 2006 Toni Ronkko
| OTHER DEALINGS IN THE SOFTWARE.
----------------------------------------------------------------------
-License for simplejson used in the python implementation:
-
-Source from: https://github.com/simplejson/simplejson
-
-Copyright (c) 2006 Bob Ippolito
-
-| Permission is hereby granted, free of charge, to any person obtaining a copy of
-| this software and associated documentation files (the "Software"), to deal in
-| the Software without restriction, including without limitation the rights to
-| use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-| of the Software, and to permit persons to whom the Software is furnished to do
-| so, subject to the following conditions:
-|
-| The above copyright notice and this permission notice shall be included in all
-| copies or substantial portions of the Software.
-|
-| THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-| IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-| FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-| AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-| LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-| OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-| SOFTWARE.
-
-----------------------------------------------------------------------
License for ivy-2.2.0.jar used in the python implementation:
Apache License version 2.0 (see above)
diff --git a/lang/py/.gitignore b/lang/py/.gitignore
index 44d9b58..ef29019 100644
--- a/lang/py/.gitignore
+++ b/lang/py/.gitignore
@@ -1,4 +1,5 @@
+*.egg-info/
+.eggs/
build/
lib/
userlogs/
-*.egg-info/
diff --git a/lang/py/build.sh b/lang/py/build.sh
index 662d854..33040d8 100755
--- a/lang/py/build.sh
+++ b/lang/py/build.sh
@@ -28,8 +28,7 @@ main() {
for target; do
case "$target" in
lint)
- ./setup.py isort
- pycodestyle .
+ ./setup.py isort lint
;;
test)
ant test
diff --git a/lang/py/setup.cfg b/lang/py/setup.cfg
index dc07a1d..1f6fbfa 100644
--- a/lang/py/setup.cfg
+++ b/lang/py/setup.cfg
@@ -19,7 +19,7 @@ line_length = 150
known_third_party=zope
[pycodestyle]
+exclude = .eggs,build
+ignore = E101,E111,E114,E121,E122,E124,E125,E126,E127,E128,E129,E201,E202,E203,E222,E226,E225,E231,E241,E251,E261,E262,E265,E266,E301,E302,E303,E305,E306,E402,E501,E701,E703,E704,E711,E722,W191,W291,W292,W293,W391,W503,W504,W601
max-line-length = 150
statistics = True
-exclude = build
-ignore = E101,E111,E114,E121,E122,E124,E125,E126,E127,E128,E129,E201,E202,E203,E222,E226,E225,E231,E241,E251,E261,E262,E265,E266,E301,E302,E303,E305,E306,E402,E501,E701,E703,E704,E711,W191,W291,W292,W293,W391,W503,W601
diff --git a/lang/py/setup.py b/lang/py/setup.py
index 6902116..6063869 100755
--- a/lang/py/setup.py
+++ b/lang/py/setup.py
@@ -16,15 +16,61 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+
+import distutils.errors
+import glob
+import os
+import subprocess
+
import setuptools
+def _get_version():
+ curdir = os.getcwd()
+ version_file = ("VERSION.txt" if os.path.isfile("VERSION.txt")
+ else os.path.join(curdir[:curdir.index("lang/py")], "share/VERSION.txt"))
+ with open(version_file) as verfile:
+ # To follow the naming convention defined by PEP 440
+ # in the case that the version is like "x.y.z-SNAPSHOT"
+ return verfile.read().rstrip().replace("-", "+")
+
+
+class LintCommand(setuptools.Command):
+ """Run pycodestyle on all your modules"""
+ description = __doc__
+ user_options = []
+
+ def initialize_options(self):
+ pass
+
+ def finalize_options(self):
+ pass
+
+ def run(self):
+ # setuptools does not seem to make pycodestyle available
+ # in the pythonpath, so we do it ourselves.
+ try:
+ env = {'PYTHONPATH': next(glob.iglob('.eggs/pycodestyle-*.egg'))}
+ except StopIteration:
+ env = None # pycodestyle is already installed
+ p = subprocess.Popen(['python', '-m', 'pycodestyle', '.'], close_fds=True, env=env)
+ if p.wait():
+ raise distutils.errors.DistutilsError("pycodestyle exited with a nonzero exit code.")
+
+
setuptools.setup(
name = 'avro',
- version = '@AVRO_VERSION@',
+ version = _get_version(),
packages = ['avro'],
- package_dir = {'avro': 'src/avro'},
+ package_dir = {'': 'src'},
scripts = ["./scripts/avro"],
+ setup_requires = [
+ 'isort',
+ 'pycodestyle',
+ ],
+ cmdclass={
+ "lint": LintCommand,
+ },
#include_package_data=True,
package_data={'avro': ['LICENSE', 'NOTICE']},
diff --git a/lang/py/src/avro/protocol.py b/lang/py/src/avro/protocol.py
index 20ce2d5..f5c333b 100644
--- a/lang/py/src/avro/protocol.py
+++ b/lang/py/src/avro/protocol.py
@@ -24,6 +24,7 @@ except ImportError:
from md5 import md5
import json
+
from avro import schema
#
diff --git a/lang/py/src/avro/tether/tether_task_runner.py b/lang/py/src/avro/tether/tether_task_runner.py
index 33d224a..b248ffd 100644
--- a/lang/py/src/avro/tether/tether_task_runner.py
+++ b/lang/py/src/avro/tether/tether_task_runner.py
@@ -96,7 +96,7 @@ class TaskRunnerResponder(ipc.Responder):
def HTTPHandlerGen(runner):
"""
This is a class factory for the HTTPHandler. We need
- a factory b\c we need a reference to the runner
+ a factory because we need a reference to the runner
Parameters
-----------------------------------------------------------------
diff --git a/lang/py3/.gitignore b/lang/py3/.gitignore
index 777f026..9714b18 100644
--- a/lang/py3/.gitignore
+++ b/lang/py3/.gitignore
@@ -1,3 +1,4 @@
+.eggs/
avro/HandshakeRequest.avsc
avro/HandshakeResponse.avsc
avro/VERSION.txt
diff --git a/lang/py3/setup.cfg b/lang/py3/setup.cfg
index 108c740..89b17fe 100644
--- a/lang/py3/setup.cfg
+++ b/lang/py3/setup.cfg
@@ -42,6 +42,9 @@ package_dir =
avro = avro
include_package_data = true
packages = avro
+setup_requires =
+ isort
+ pycodestyle
install_requires =
zip_safe = true
scripts =
@@ -68,6 +71,7 @@ line_length = 150
known_third_party=zope
[pycodestyle]
+exclude = .eggs,build
+ignore = E111,E114,E121,E122,E124,E127,E128,E129,E201,E202,E203,E221,E225,E231,E241,E261,E301,E302,E303,E305,E402,E701,E703,E722,W503,W504
max-line-length = 150
statistics = True
-ignore = E111,E114,E121,E122,E124,E127,E128,E129,E201,E202,E203,E221,E225,E231,E241,E261,E301,E302,E303,E305,E402,E701,E703,W503
diff --git a/lang/py3/setup.py b/lang/py3/setup.py
index 130ce18..20dd4e2 100755
--- a/lang/py3/setup.py
+++ b/lang/py3/setup.py
@@ -32,6 +32,7 @@ import distutils.errors
import distutils.file_util
import distutils.log
import fnmatch
+import glob
import os
import subprocess
@@ -146,8 +147,14 @@ class LintCommand(setuptools.Command):
pass
def run(self):
+ # setuptools does not seem to make pycodestyle available
+ # in the pythonpath, so we do it ourselves.
try:
- subprocess.run(['pycodestyle', '.'], check=True)
+ env = {'PYTHONPATH': next(glob.iglob('.eggs/pycodestyle-*.egg'))}
+ except StopIteration:
+ env = None # pycodestyle is already installed
+ try:
+ subprocess.run(['python3', '-m', 'pycodestyle', '.'], env=env)
except subprocess.CalledProcessError:
raise distutils.errors.DistutilsError("pycodestyle exited with a nonzero exit code.")
diff --git a/pom.xml b/pom.xml
index bb4f8bc..631fa60 100644
--- a/pom.xml
+++ b/pom.xml
@@ -49,15 +49,15 @@
<!-- plugin versions -->
<antrun-plugin.version>1.8</antrun-plugin.version>
- <checkstyle-plugin.version>3.0.0</checkstyle-plugin.version>
+ <checkstyle-plugin.version>3.1.0</checkstyle-plugin.version>
<enforcer-plugin.version>3.0.0-M2</enforcer-plugin.version>
<extra-enforcer-rules.version>1.2</extra-enforcer-rules.version>
<gpg-plugin.version>1.6</gpg-plugin.version>
- <javadoc-plugin.version>3.1.0</javadoc-plugin.version>
+ <javadoc-plugin.version>3.1.1</javadoc-plugin.version>
<maven-plugin-plugin.version>3.6.0</maven-plugin-plugin.version>
<rat.version>0.13</rat.version>
<source-plugin.version>3.0.1</source-plugin.version>
- <spotless-maven-plugin.version>1.21.1</spotless-maven-plugin.version>
+ <spotless-maven-plugin.version>1.25.1</spotless-maven-plugin.version>
</properties>
<modules>
@@ -290,6 +290,7 @@
<!-- build or test files (some are generated files that we commit as-is for testing purposes) -->
<exclude>**/*.log</exclude>
<exclude>**/*.rej</exclude>
+ <exclude>**/*.egg-info/**</exclude>
<exclude>build/**</exclude>
<exclude>doc/build/**</exclude>
<exclude>lang/java/archetypes/**</exclude>
@@ -301,7 +302,6 @@
<exclude>lang/perl/pm_to_blib</exclude>
<exclude>lang/perl/blib/**/.exists</exclude>
<exclude>lang/py/build/**</exclude>
- <exclude>lang/py3/avro_python3.egg-info/**</exclude>
<exclude>lang/ruby/Gemfile.lock</exclude>
<exclude>lang/ruby/avro.gemspec</exclude>
<exclude>lang/ruby/.gem/**</exclude>
@@ -351,36 +351,42 @@
<exclude>lang/csharp/Avro.sln</exclude> <!-- visual studio -->
<!-- build-related files -->
<exclude>**/README.md</exclude>
- <exclude>BUILD.md</exclude>
- <exclude>.travis.yml</exclude>
<exclude>**/VERSION.txt</exclude>
<exclude>**/dependency-reduced-pom.xml</exclude>
+ <exclude>.travis.yml</exclude>
+ <exclude>BUILD.md</exclude>
+ <exclude>lang/c/src/avro-c.pc.in</exclude>
+ <exclude>lang/csharp/**/bin/Release/**/Avro.xml</exclude>
+ <exclude>lang/csharp/TestResult.xml</exclude>
+ <exclude>lang/csharp/src/apache/*/obj/**</exclude>
+ <exclude>lang/java/archetypes/avro-service-archetype/src/test/integration/projects/basic/goal.txt</exclude>
+ <exclude>lang/java/mapred/userlogs/**</exclude>
+ <exclude>lang/java/tools/userlogs/**</exclude>
+ <exclude>lang/js/coverage/**</exclude>
+ <exclude>lang/js/test/mocha.opts</exclude>
<exclude>lang/perl/.shipit</exclude>
- <exclude>lang/perl/inc/Module/Install/*.pm</exclude>
- <exclude>lang/perl/inc/Module/Install.pm</exclude>
- <exclude>lang/perl/Makefile*</exclude>
<exclude>lang/perl/META.yml</exclude>
<exclude>lang/perl/MYMETA.yml</exclude>
+ <exclude>lang/perl/Makefile*</exclude>
+ <exclude>lang/perl/inc/Module/Install.pm</exclude>
+ <exclude>lang/perl/inc/Module/Install/*.pm</exclude>
+ <exclude>lang/py/.eggs/**</exclude>
+ <exclude>lang/py/build/**</exclude>
+ <exclude>lang/py/dist/**</exclude>
<exclude>lang/py/userlogs/**</exclude>
- <exclude>lang/c/src/avro-c.pc.in</exclude>
+ <exclude>lang/py3/.eggs/**</exclude>
+ <exclude>lang/py3/build/**</exclude>
+ <exclude>lang/py3/dist/**</exclude>
<exclude>lang/ruby/Manifest</exclude>
- <exclude>lang/java/tools/userlogs/**</exclude>
- <exclude>lang/java/mapred/userlogs/**</exclude>
- <exclude>lang/java/archetypes/avro-service-archetype/src/test/integration/projects/basic/goal.txt</exclude>
- <exclude>lang/js/test/mocha.opts</exclude>
- <exclude>lang/csharp/TestResult.xml</exclude>
- <exclude>lang/csharp/src/apache/*/obj/**</exclude>
- <exclude>lang/csharp/**/bin/Release/**/Avro.xml</exclude>
- <exclude>lang/js/coverage/**</exclude>
<!-- text documentation files -->
<exclude>CHANGES.txt</exclude>
<exclude>DIST_README.txt</exclude>
<exclude>lang/perl/Changes</exclude>
<exclude>lang/c/README.maintaining_win32.txt</exclude>
<exclude>lang/c/docs/index.txt</exclude>
+ <exclude>.github/PULL_REQUEST_TEMPLATE.md</exclude>
<exclude>lang/java/archetypes/avro-service-archetype/src/test/integration/projects/basic/archetype.properties</exclude> <!-- used to generate user projects -->
<!-- files and directories covered by LICENSE.txt -->
- <exclude>lang/py/lib/simplejson/**</exclude>
<exclude>lang/c/jansson/**</exclude>
<exclude>lang/c/src/avro/msinttypes.h</exclude>
<exclude>lang/c/src/avro/msstdint.h</exclude>
diff --git a/share/docker/Dockerfile b/share/docker/Dockerfile
index c6944ce..726d321 100644
--- a/share/docker/Dockerfile
+++ b/share/docker/Dockerfile
@@ -53,7 +53,6 @@ RUN apt-get -qq update && \
g++ \
gcc \
git \
- isort \
libboost-all-dev \
libfontconfig1-dev \
libfreetype6-dev \
@@ -69,14 +68,14 @@ RUN apt-get -qq update && \
php5.6-gmp \
pycodestyle \
python \
- python-isort \
python-pip \
python-setuptools \
python-snappy \
python-wheel \
- python3-isort \
python3-pip \
python3-setuptools \
+ python3-snappy \
+ python3-wheel \
rake \
ruby \
ruby-dev \
@@ -98,6 +97,14 @@ RUN curl -L https://cpanmin.us | perl - --self-upgrade && \
# Install PHPUnit
RUN wget -O /usr/local/bin/phpunit https://phar.phpunit.de/phpunit-5.6.phar && chmod +x /usr/local/bin/phpunit
+# Install Python2 packages
+RUN python2 -m pip install --upgrade pip setuptools \
+ && python2 -m pip install zstandard
+
+# Install Python3 packages
+RUN python3 -m pip install --upgrade pip setuptools \
+ && python3 -m pip install zstandard
+
# Install Ruby modules
RUN gem install echoe yajl-ruby multi_json snappy
[avro] 12/23: AVRO-2578: Support Both Capitalizations of Parse
(#666)
Posted by rs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rskraba pushed a commit to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/avro.git
commit 78f26ccb3fef97bda59250cf19ff9e9c5f2104bf
Author: Michael A. Smith <mi...@smith-li.com>
AuthorDate: Sat Oct 26 21:27:12 2019 -0400
AVRO-2578: Support Both Capitalizations of Parse (#666)
* AVRO-2578: Provide Parse in Both Capitalizations
* AVRO-2578: Normalize Calls to Parse
* AVRO-2578: Test Both Schema Parse Capitalizations
* AVRO-2578: Test Both Capitalizations of Protocol.Parse
* AVRO-2578: Add Deprecation Warnings for Capital-P Parse
* AVRO-2578: Fix Tests for Py3.5
---
lang/py3/avro/datafile.py | 6 +-
lang/py3/avro/ipc.py | 14 +-
lang/py3/avro/protocol.py | 10 +-
lang/py3/avro/schema.py | 11 +-
lang/py3/avro/tests/av_bench.py | 2 +-
lang/py3/avro/tests/gen_interop_data.py | 24 ++-
lang/py3/avro/tests/sample_http_client.py | 2 +-
lang/py3/avro/tests/sample_http_server.py | 2 +-
lang/py3/avro/tests/test_datafile.py | 8 +-
lang/py3/avro/tests/test_io.py | 30 +--
lang/py3/avro/tests/test_ipc.py | 2 +-
lang/py3/avro/tests/test_normalization.py | 280 ++++++++++++++--------------
lang/py3/avro/tests/test_protocol.py | 22 ++-
lang/py3/avro/tests/test_schema.py | 35 ++--
lang/py3/avro/tests/test_script.py | 2 +-
lang/py3/avro/tests/txsample_http_client.py | 2 +-
lang/py3/avro/tests/txsample_http_server.py | 2 +-
lang/py3/avro/tool.py | 40 ++--
lang/py3/scripts/avro | 2 +-
19 files changed, 266 insertions(+), 230 deletions(-)
diff --git a/lang/py3/avro/datafile.py b/lang/py3/avro/datafile.py
index 1e9b3fa..8746ef7 100644
--- a/lang/py3/avro/datafile.py
+++ b/lang/py3/avro/datafile.py
@@ -57,7 +57,7 @@ SYNC_SIZE = 16
SYNC_INTERVAL = 1000 * SYNC_SIZE
# Schema of the container header:
-META_SCHEMA = schema.Parse("""
+META_SCHEMA = schema.parse("""
{
"type": "record", "name": "org.apache.avro.file.Header",
"fields": [{
@@ -164,7 +164,7 @@ class DataFileWriter(object):
# get schema used to write existing file
schema_from_file = dfr.GetMeta('avro.schema').decode('utf-8')
self.SetMeta('avro.schema', schema_from_file)
- self.datum_writer.writer_schema = schema.Parse(schema_from_file)
+ self.datum_writer.writer_schema = schema.parse(schema_from_file)
# seek to the end of the file and prepare for writing
writer.seek(0, 2)
@@ -364,7 +364,7 @@ class DataFileReader(object):
# get ready to read
self._block_count = 0
self.datum_reader.writer_schema = (
- schema.Parse(self.GetMeta(SCHEMA_KEY).decode('utf-8')))
+ schema.parse(self.GetMeta(SCHEMA_KEY).decode('utf-8')))
def __enter__(self):
return self
diff --git a/lang/py3/avro/ipc.py b/lang/py3/avro/ipc.py
index f5d376d..22b9b11 100644
--- a/lang/py3/avro/ipc.py
+++ b/lang/py3/avro/ipc.py
@@ -47,19 +47,19 @@ def LoadResource(name):
HANDSHAKE_REQUEST_SCHEMA_JSON = LoadResource('HandshakeRequest.avsc')
HANDSHAKE_RESPONSE_SCHEMA_JSON = LoadResource('HandshakeResponse.avsc')
-HANDSHAKE_REQUEST_SCHEMA = schema.Parse(HANDSHAKE_REQUEST_SCHEMA_JSON)
-HANDSHAKE_RESPONSE_SCHEMA = schema.Parse(HANDSHAKE_RESPONSE_SCHEMA_JSON)
+HANDSHAKE_REQUEST_SCHEMA = schema.parse(HANDSHAKE_REQUEST_SCHEMA_JSON)
+HANDSHAKE_RESPONSE_SCHEMA = schema.parse(HANDSHAKE_RESPONSE_SCHEMA_JSON)
HANDSHAKE_REQUESTOR_WRITER = avro_io.DatumWriter(HANDSHAKE_REQUEST_SCHEMA)
HANDSHAKE_REQUESTOR_READER = avro_io.DatumReader(HANDSHAKE_RESPONSE_SCHEMA)
HANDSHAKE_RESPONDER_WRITER = avro_io.DatumWriter(HANDSHAKE_RESPONSE_SCHEMA)
HANDSHAKE_RESPONDER_READER = avro_io.DatumReader(HANDSHAKE_REQUEST_SCHEMA)
-META_SCHEMA = schema.Parse('{"type": "map", "values": "bytes"}')
+META_SCHEMA = schema.parse('{"type": "map", "values": "bytes"}')
META_WRITER = avro_io.DatumWriter(META_SCHEMA)
META_READER = avro_io.DatumReader(META_SCHEMA)
-SYSTEM_ERROR_SCHEMA = schema.Parse('["string"]')
+SYSTEM_ERROR_SCHEMA = schema.parse('["string"]')
AVRO_RPC_MIME = 'avro/binary'
@@ -225,7 +225,7 @@ class BaseRequestor(object, metaclass=abc.ABCMeta):
elif match == 'CLIENT':
# Client's side hash mismatch:
self._remote_protocol = \
- protocol.Parse(handshake_response['serverProtocol'])
+ protocol.parse(handshake_response['serverProtocol'])
self._remote_hash = handshake_response['serverHash']
self._send_protocol = False
return True
@@ -233,7 +233,7 @@ class BaseRequestor(object, metaclass=abc.ABCMeta):
elif match == 'NONE':
# Neither client nor server match:
self._remote_protocol = \
- protocol.Parse(handshake_response['serverProtocol'])
+ protocol.parse(handshake_response['serverProtocol'])
self._remote_hash = handshake_response['serverHash']
self._send_protocol = True
return False
@@ -415,7 +415,7 @@ class Responder(object, metaclass=abc.ABCMeta):
client_protocol = handshake_request.get('clientProtocol')
remote_protocol = self.get_protocol_cache(client_hash)
if remote_protocol is None and client_protocol is not None:
- remote_protocol = protocol.Parse(client_protocol)
+ remote_protocol = protocol.parse(client_protocol)
self.set_protocol_cache(client_hash, remote_protocol)
# evaluate remote's guess of the local protocol
diff --git a/lang/py3/avro/protocol.py b/lang/py3/avro/protocol.py
index 30834da..8ee721c 100644
--- a/lang/py3/avro/protocol.py
+++ b/lang/py3/avro/protocol.py
@@ -24,6 +24,7 @@ Protocol implementation.
import hashlib
import json
import logging
+import warnings
from types import MappingProxyType
from avro import schema
@@ -375,7 +376,7 @@ def ProtocolFromJSONData(json_data):
)
-def Parse(json_string):
+def parse(json_string):
"""Constructs a Protocol from its JSON descriptor in text form.
Args:
@@ -395,3 +396,10 @@ def Parse(json_string):
% (json_string, exn))
return ProtocolFromJSONData(json_data)
+
+def Parse(json_string):
+ """Deprecated implementation of parse."""
+ warnings.warn("`Parse` is deprecated in avro 1.10. "
+ "Please use `parse` (lowercase) instead.",
+ DeprecationWarning)
+ return parse(json_string)
diff --git a/lang/py3/avro/schema.py b/lang/py3/avro/schema.py
index 9fbe8e5..2bfaa3b 100644
--- a/lang/py3/avro/schema.py
+++ b/lang/py3/avro/schema.py
@@ -42,6 +42,7 @@ import abc
import json
import logging
import re
+import warnings
from types import MappingProxyType
logger = logging.getLogger(__name__)
@@ -1217,7 +1218,7 @@ def SchemaFromJSONData(json_data, names=None):
# ------------------------------------------------------------------------------
-def Parse(json_string):
+def parse(json_string):
"""Constructs a Schema from its JSON descriptor in text form.
Args:
@@ -1241,3 +1242,11 @@ def Parse(json_string):
# construct the Avro Schema object
return SchemaFromJSONData(json_data, names)
+
+
+def Parse(json_string):
+ """Deprecated implementation of parse."""
+ warnings.warn("`Parse` is deprecated in avro 1.10. "
+ "Please use `parse` (lowercase) instead.",
+ DeprecationWarning)
+ return parse(json_string)
diff --git a/lang/py3/avro/tests/av_bench.py b/lang/py3/avro/tests/av_bench.py
index 88cbafb..d4baca2 100644
--- a/lang/py3/avro/tests/av_bench.py
+++ b/lang/py3/avro/tests/av_bench.py
@@ -62,7 +62,7 @@ def Write(nrecords):
]
}
"""
- schema = avro.schema.Parse(schema_s)
+ schema = avro.schema.parse(schema_s)
writer = avro.io.DatumWriter(schema)
with open(FILENAME, 'wb') as out:
diff --git a/lang/py3/avro/tests/gen_interop_data.py b/lang/py3/avro/tests/gen_interop_data.py
index c76f2c2..f3ee9be 100644
--- a/lang/py3/avro/tests/gen_interop_data.py
+++ b/lang/py3/avro/tests/gen_interop_data.py
@@ -19,8 +19,10 @@
# limitations under the License.
import sys
+from pathlib import Path
from avro import datafile, io, schema
+from avro.datafile import NULL_CODEC
DATUM = {
'intField': 12,
@@ -29,7 +31,7 @@ DATUM = {
'boolField': True,
'floatField': 1234.0,
'doubleField': -1234.0,
- 'bytesField': '12312adf',
+ 'bytesField': b'12312adf',
'nullField': None,
'arrayField': [5.0, 0.0, 12.0],
'mapField': {'a': {'label': 'a'}, 'bee': {'label': 'cee'}},
@@ -43,11 +45,17 @@ DATUM = {
}
-if __name__ == "__main__":
- interop_schema = schema.Parse(open(sys.argv[1], 'r').read())
- writer = open(sys.argv[2], 'wb')
+def generate(schema_file, output_path):
+ interop_schema = schema.parse(open(schema_file, 'r').read())
datum_writer = io.DatumWriter()
- # NB: not using compression
- dfw = datafile.DataFileWriter(writer, datum_writer, interop_schema)
- dfw.append(DATUM)
- dfw.close()
+ for codec in datafile.VALID_CODECS:
+ filename = 'py3'
+ if codec != NULL_CODEC:
+ filename += '_' + codec
+ with Path(output_path, filename).with_suffix('.avro').open('wb') as writer, \
+ datafile.DataFileWriter(writer, datum_writer, interop_schema, codec) as dfw:
+ dfw.append(DATUM)
+
+
+if __name__ == "__main__":
+ generate(sys.argv[1], sys.argv[2])
diff --git a/lang/py3/avro/tests/sample_http_client.py b/lang/py3/avro/tests/sample_http_client.py
index 47212a1..379389a 100644
--- a/lang/py3/avro/tests/sample_http_client.py
+++ b/lang/py3/avro/tests/sample_http_client.py
@@ -47,7 +47,7 @@ MAIL_PROTOCOL_JSON = """\
}
}
"""
-MAIL_PROTOCOL = protocol.Parse(MAIL_PROTOCOL_JSON)
+MAIL_PROTOCOL = protocol.parse(MAIL_PROTOCOL_JSON)
SERVER_HOST = 'localhost'
SERVER_PORT = 9090
diff --git a/lang/py3/avro/tests/sample_http_server.py b/lang/py3/avro/tests/sample_http_server.py
index 72518be..22aaf6f 100644
--- a/lang/py3/avro/tests/sample_http_server.py
+++ b/lang/py3/avro/tests/sample_http_server.py
@@ -48,7 +48,7 @@ MAIL_PROTOCOL_JSON = """\
}
}
"""
-MAIL_PROTOCOL = protocol.Parse(MAIL_PROTOCOL_JSON)
+MAIL_PROTOCOL = protocol.parse(MAIL_PROTOCOL_JSON)
SERVER_ADDRESS = ('localhost', 9090)
class MailResponder(ipc.Responder):
diff --git a/lang/py3/avro/tests/test_datafile.py b/lang/py3/avro/tests/test_datafile.py
index aa4042f..2ab1ab0 100644
--- a/lang/py3/avro/tests/test_datafile.py
+++ b/lang/py3/avro/tests/test_datafile.py
@@ -144,7 +144,7 @@ class TestDataFile(unittest.TestCase):
logging.debug('Creating data file %r', file_path)
with open(file_path, 'wb') as writer:
datum_writer = io.DatumWriter()
- schema_object = schema.Parse(writer_schema)
+ schema_object = schema.parse(writer_schema)
with datafile.DataFileWriter(
writer=writer,
datum_writer=datum_writer,
@@ -193,7 +193,7 @@ class TestDataFile(unittest.TestCase):
logging.debug('Creating data file %r', file_path)
with open(file_path, 'wb') as writer:
datum_writer = io.DatumWriter()
- schema_object = schema.Parse(writer_schema)
+ schema_object = schema.parse(writer_schema)
with datafile.DataFileWriter(
writer=writer,
datum_writer=datum_writer,
@@ -239,7 +239,7 @@ class TestDataFile(unittest.TestCase):
with open(file_path, 'wb') as writer:
datum_writer = io.DatumWriter()
sample_schema, sample_datum = SCHEMAS_TO_VALIDATE[1]
- schema_object = schema.Parse(sample_schema)
+ schema_object = schema.parse(sample_schema)
with datafile.DataFileWriter(writer, datum_writer, schema_object) as dfw:
dfw.append(sample_datum)
self.assertTrue(writer.closed)
@@ -260,7 +260,7 @@ class TestDataFile(unittest.TestCase):
with open(file_path, 'wb') as writer:
datum_writer = io.DatumWriter()
sample_schema, sample_datum = SCHEMAS_TO_VALIDATE[1]
- schema_object = schema.Parse(sample_schema)
+ schema_object = schema.parse(sample_schema)
with datafile.DataFileWriter(writer, datum_writer, schema_object) as dfw:
dfw.SetMeta('test.string', 'foo')
dfw.SetMeta('test.number', '1')
diff --git a/lang/py3/avro/tests/test_io.py b/lang/py3/avro/tests/test_io.py
index 2fcd689..cbd2219 100644
--- a/lang/py3/avro/tests/test_io.py
+++ b/lang/py3/avro/tests/test_io.py
@@ -97,7 +97,7 @@ DEFAULT_VALUE_EXAMPLES = (
'{"A": 5}', {'A': 5}),
)
-LONG_RECORD_SCHEMA = schema.Parse("""
+LONG_RECORD_SCHEMA = schema.parse("""
{
"type": "record",
"name": "Test",
@@ -149,7 +149,7 @@ def check_binary_encoding(number_type):
logging.debug('Datum: %d', datum)
logging.debug('Correct Encoding: %s', hex_encoding)
- writer_schema = schema.Parse('"%s"' % number_type.lower())
+ writer_schema = schema.parse('"%s"' % number_type.lower())
writer, encoder, datum_writer = write_datum(datum, writer_schema)
writer.seek(0)
hex_val = avro_hexlify(writer)
@@ -167,7 +167,7 @@ def check_skip_number(number_type):
logging.debug('Value to Skip: %d', value_to_skip)
# write the value to skip and a known value
- writer_schema = schema.Parse('"%s"' % number_type.lower())
+ writer_schema = schema.parse('"%s"' % number_type.lower())
writer, encoder, datum_writer = write_datum(value_to_skip, writer_schema)
datum_writer.write(VALUE_TO_READ, encoder)
@@ -198,7 +198,7 @@ class TestIO(unittest.TestCase):
for example_schema, datum in SCHEMAS_TO_VALIDATE:
logging.debug('Schema: %r', example_schema)
logging.debug('Datum: %r', datum)
- validated = avro_io.Validate(schema.Parse(example_schema), datum)
+ validated = avro_io.Validate(schema.parse(example_schema), datum)
logging.debug('Valid: %s', validated)
if validated: passed += 1
self.assertEqual(passed, len(SCHEMAS_TO_VALIDATE))
@@ -209,7 +209,7 @@ class TestIO(unittest.TestCase):
logging.debug('Schema: %s', example_schema)
logging.debug('Datum: %s', datum)
- writer_schema = schema.Parse(example_schema)
+ writer_schema = schema.parse(example_schema)
writer, encoder, datum_writer = write_datum(datum, writer_schema)
round_trip_datum = read_datum(writer, writer_schema)
@@ -247,10 +247,10 @@ class TestIO(unittest.TestCase):
promotable_schemas = ['"int"', '"long"', '"float"', '"double"']
incorrect = 0
for i, ws in enumerate(promotable_schemas):
- writer_schema = schema.Parse(ws)
+ writer_schema = schema.parse(ws)
datum_to_write = 219
for rs in promotable_schemas[i + 1:]:
- reader_schema = schema.Parse(rs)
+ reader_schema = schema.parse(rs)
writer, enc, dw = write_datum(datum_to_write, writer_schema)
datum_read = read_datum(writer, writer_schema, reader_schema)
logging.debug('Writer: %s Reader: %s', writer_schema, reader_schema)
@@ -259,12 +259,12 @@ class TestIO(unittest.TestCase):
self.assertEqual(incorrect, 0)
def testUnknownSymbol(self):
- writer_schema = schema.Parse("""\
+ writer_schema = schema.parse("""\
{"type": "enum", "name": "Test",
"symbols": ["FOO", "BAR"]}""")
datum_to_write = 'FOO'
- reader_schema = schema.Parse("""\
+ reader_schema = schema.parse("""\
{"type": "enum", "name": "Test",
"symbols": ["BAR", "BAZ"]}""")
@@ -280,7 +280,7 @@ class TestIO(unittest.TestCase):
correct = 0
for field_type, default_json, default_datum in DEFAULT_VALUE_EXAMPLES:
- reader_schema = schema.Parse("""\
+ reader_schema = schema.parse("""\
{"type": "record", "name": "Test",
"fields": [{"name": "H", "type": %s, "default": %s}]}
""" % (field_type, default_json))
@@ -296,7 +296,7 @@ class TestIO(unittest.TestCase):
writer_schema = LONG_RECORD_SCHEMA
datum_to_write = LONG_RECORD_DATUM
- reader_schema = schema.Parse("""\
+ reader_schema = schema.parse("""\
{"type": "record", "name": "Test",
"fields": [{"name": "H", "type": "int"}]}""")
@@ -310,7 +310,7 @@ class TestIO(unittest.TestCase):
writer_schema = LONG_RECORD_SCHEMA
datum_to_write = LONG_RECORD_DATUM
- reader_schema = schema.Parse("""\
+ reader_schema = schema.parse("""\
{"type": "record", "name": "Test",
"fields": [{"name": "E", "type": "int"},
{"name": "F", "type": "int"}]}""")
@@ -325,7 +325,7 @@ class TestIO(unittest.TestCase):
writer_schema = LONG_RECORD_SCHEMA
datum_to_write = LONG_RECORD_DATUM
- reader_schema = schema.Parse("""\
+ reader_schema = schema.parse("""\
{"type": "record", "name": "Test",
"fields": [{"name": "F", "type": "int"},
{"name": "E", "type": "int"}]}""")
@@ -337,7 +337,7 @@ class TestIO(unittest.TestCase):
self.assertEqual(datum_to_read, datum_read)
def testTypeException(self):
- writer_schema = schema.Parse("""\
+ writer_schema = schema.parse("""\
{"type": "record", "name": "Test",
"fields": [{"name": "F", "type": "int"},
{"name": "E", "type": "int"}]}""")
@@ -346,7 +346,7 @@ class TestIO(unittest.TestCase):
avro_io.AvroTypeException, write_datum, datum_to_write, writer_schema)
def testUnionSchemaSpecificity(self):
- union_schema = schema.Parse("""
+ union_schema = schema.parse("""
[{
"type" : "record",
"name" : "A",
diff --git a/lang/py3/avro/tests/test_ipc.py b/lang/py3/avro/tests/test_ipc.py
index 20eee1a..fa23d3c 100644
--- a/lang/py3/avro/tests/test_ipc.py
+++ b/lang/py3/avro/tests/test_ipc.py
@@ -75,7 +75,7 @@ ECHO_PROTOCOL_JSON = """
"""
-ECHO_PROTOCOL = protocol.Parse(ECHO_PROTOCOL_JSON)
+ECHO_PROTOCOL = protocol.parse(ECHO_PROTOCOL_JSON)
class EchoResponder(ipc.Responder):
diff --git a/lang/py3/avro/tests/test_normalization.py b/lang/py3/avro/tests/test_normalization.py
index 148bfc0..f464dd7 100644
--- a/lang/py3/avro/tests/test_normalization.py
+++ b/lang/py3/avro/tests/test_normalization.py
@@ -20,7 +20,7 @@
import unittest
-from avro.schema import Parse
+from avro.schema import parse
from avro.schemanormalization import Fingerprint, FingerprintAlgorithmNames, ToParsingCanonicalForm
@@ -29,119 +29,119 @@ class TestSchemaNormalization(unittest.TestCase):
def testCanonicalization1(self):
pre='"float"'
post='"float"'
- pre_schema = Parse(pre)
+ pre_schema = parse(pre)
canonical_form = ToParsingCanonicalForm(pre_schema)
self.assertEqual(canonical_form, post)
def testCanonicalization2(self):
pre='{"type": "float"}'
post='"float"'
- pre_schema = Parse(pre)
+ pre_schema = parse(pre)
canonical_form = ToParsingCanonicalForm(pre_schema)
self.assertEqual(canonical_form, post)
def testCanonicalization3(self):
pre='"int"'
post='"int"'
- pre_schema = Parse(pre)
+ pre_schema = parse(pre)
canonical_form = ToParsingCanonicalForm(pre_schema)
self.assertEqual(canonical_form, post)
def testCanonicalization4(self):
pre='{"type": "int"}'
post='"int"'
- pre_schema = Parse(pre)
+ pre_schema = parse(pre)
canonical_form = ToParsingCanonicalForm(pre_schema)
self.assertEqual(canonical_form, post)
def testCanonicalization5(self):
pre='"double"'
post='"double"'
- pre_schema = Parse(pre)
+ pre_schema = parse(pre)
canonical_form = ToParsingCanonicalForm(pre_schema)
self.assertEqual(canonical_form, post)
def testCanonicalization6(self):
pre='{"type": "double"}'
post='"double"'
- pre_schema = Parse(pre)
+ pre_schema = parse(pre)
canonical_form = ToParsingCanonicalForm(pre_schema)
self.assertEqual(canonical_form, post)
def testCanonicalization7(self):
pre='"null"'
post='"null"'
- pre_schema = Parse(pre)
+ pre_schema = parse(pre)
canonical_form = ToParsingCanonicalForm(pre_schema)
self.assertEqual(canonical_form, post)
def testCanonicalization8(self):
pre='{"type": "null"}'
post='"null"'
- pre_schema = Parse(pre)
+ pre_schema = parse(pre)
canonical_form = ToParsingCanonicalForm(pre_schema)
self.assertEqual(canonical_form, post)
def testCanonicalization9(self):
pre='"bytes"'
post='"bytes"'
- pre_schema = Parse(pre)
+ pre_schema = parse(pre)
canonical_form = ToParsingCanonicalForm(pre_schema)
self.assertEqual(canonical_form, post)
def testCanonicalization10(self):
pre='{"type": "bytes"}'
post='"bytes"'
- pre_schema = Parse(pre)
+ pre_schema = parse(pre)
canonical_form = ToParsingCanonicalForm(pre_schema)
self.assertEqual(canonical_form, post)
def testCanonicalization11(self):
pre='"long"'
post='"long"'
- pre_schema = Parse(pre)
+ pre_schema = parse(pre)
canonical_form = ToParsingCanonicalForm(pre_schema)
self.assertEqual(canonical_form, post)
def testCanonicalization12(self):
pre='{"type": "long"}'
post='"long"'
- pre_schema = Parse(pre)
+ pre_schema = parse(pre)
canonical_form = ToParsingCanonicalForm(pre_schema)
self.assertEqual(canonical_form, post)
def testCanonicalization13(self):
pre='"boolean"'
post='"boolean"'
- pre_schema = Parse(pre)
+ pre_schema = parse(pre)
canonical_form = ToParsingCanonicalForm(pre_schema)
self.assertEqual(canonical_form, post)
def testCanonicalization14(self):
pre='{"type": "boolean"}'
post='"boolean"'
- pre_schema = Parse(pre)
+ pre_schema = parse(pre)
canonical_form = ToParsingCanonicalForm(pre_schema)
self.assertEqual(canonical_form, post)
def testCanonicalization15(self):
pre='"string"'
post='"string"'
- pre_schema = Parse(pre)
+ pre_schema = parse(pre)
canonical_form = ToParsingCanonicalForm(pre_schema)
self.assertEqual(canonical_form, post)
def testCanonicalization16(self):
pre='{"type": "string"}'
post='"string"'
- pre_schema = Parse(pre)
+ pre_schema = parse(pre)
canonical_form = ToParsingCanonicalForm(pre_schema)
self.assertEqual(canonical_form, post)
def testCanonicalization17(self):
pre='{"type": "fixed", "name": "Test", "size": 1}'
post='{"name":"Test","type":"fixed","size":1}'
- pre_schema = Parse(pre)
+ pre_schema = parse(pre)
canonical_form = ToParsingCanonicalForm(pre_schema)
self.assertEqual(canonical_form, post)
@@ -155,21 +155,21 @@ class TestSchemaNormalization(unittest.TestCase):
}
"""
post='{"name":"org.apache.hadoop.avro.MyFixed","type":"fixed","size":1}'
- pre_schema = Parse(pre)
+ pre_schema = parse(pre)
canonical_form = ToParsingCanonicalForm(pre_schema)
self.assertEqual(canonical_form, post)
def testCanonicalization19(self):
pre='{"type": "enum", "name": "Test", "symbols": ["A", "B"]}'
post='{"name":"Test","type":"enum","symbols":["A","B"]}'
- pre_schema = Parse(pre)
+ pre_schema = parse(pre)
canonical_form = ToParsingCanonicalForm(pre_schema)
self.assertEqual(canonical_form, post)
def testCanonicalization20(self):
pre='{"type": "array", "items": "long"}'
post='{"type":"array","items":"long"}'
- pre_schema = Parse(pre)
+ pre_schema = parse(pre)
canonical_form = ToParsingCanonicalForm(pre_schema)
self.assertEqual(canonical_form, post)
@@ -182,14 +182,14 @@ class TestSchemaNormalization(unittest.TestCase):
"""
post=('{"type":"array","items":{"name":"Test'
'","type":"enum","symbols":["A","B"]}}')
- pre_schema = Parse(pre)
+ pre_schema = parse(pre)
canonical_form = ToParsingCanonicalForm(pre_schema)
self.assertEqual(canonical_form, post)
def testCanonicalization22(self):
pre='{"type": "map", "values": "long"}'
post='{"type":"map","values":"long"}'
- pre_schema = Parse(pre)
+ pre_schema = parse(pre)
canonical_form = ToParsingCanonicalForm(pre_schema)
self.assertEqual(canonical_form, post)
@@ -202,14 +202,14 @@ class TestSchemaNormalization(unittest.TestCase):
"""
post=('{"type":"map","values":{"name":"Test"'
',"type":"enum","symbols":["A","B"]}}')
- pre_schema = Parse(pre)
+ pre_schema = parse(pre)
canonical_form = ToParsingCanonicalForm(pre_schema)
self.assertEqual(canonical_form, post)
def testCanonicalization24(self):
pre='["string", "null", "long"]'
post='["string","null","long"]'
- pre_schema = Parse(pre)
+ pre_schema = parse(pre)
canonical_form = ToParsingCanonicalForm(pre_schema)
self.assertEqual(canonical_form, post)
@@ -222,7 +222,7 @@ class TestSchemaNormalization(unittest.TestCase):
}
"""
post='{"name":"Test","type":"record","fields":[{"name":"f","type":"long"}]}'
- pre_schema = Parse(pre)
+ pre_schema = parse(pre)
canonical_form = ToParsingCanonicalForm(pre_schema)
self.assertEqual(canonical_form, post)
@@ -235,7 +235,7 @@ class TestSchemaNormalization(unittest.TestCase):
}
"""
post='{"name":"Test","type":"record","fields":[{"name":"f","type":"long"}]}'
- pre_schema = Parse(pre)
+ pre_schema = parse(pre)
canonical_form = ToParsingCanonicalForm(pre_schema)
self.assertEqual(canonical_form, post)
@@ -253,7 +253,7 @@ class TestSchemaNormalization(unittest.TestCase):
post=('{"name":"Node","type":"record","fields":[{"na'
'me":"label","type":"string"},{"name":"childre'
'n","type":{"type":"array","items":"Node"}}]}')
- pre_schema = Parse(pre)
+ pre_schema = parse(pre)
canonical_form = ToParsingCanonicalForm(pre_schema)
self.assertEqual(canonical_form, post)
@@ -280,7 +280,7 @@ class TestSchemaNormalization(unittest.TestCase):
post=('{"name":"Lisp","type":"record","fields":[{"name":"value","type'
'":["null","string",{"name":"Cons","type":"record","fields":[{"'
'name":"car","type":"Lisp"},{"name":"cdr","type":"Lisp"}]}]}]}')
- pre_schema = Parse(pre)
+ pre_schema = parse(pre)
canonical_form = ToParsingCanonicalForm(pre_schema)
self.assertEqual(canonical_form, post)
@@ -310,7 +310,7 @@ class TestSchemaNormalization(unittest.TestCase):
'":"clientProtocol","type":["null","string"]},{"name":"se'
'rverHash","type":"org.apache.avro.ipc.MD5"},{"name":"met'
'a","type":["null",{"type":"map","values":"bytes"}]}]}')
- pre_schema = Parse(pre)
+ pre_schema = parse(pre)
canonical_form = ToParsingCanonicalForm(pre_schema)
self.assertEqual(canonical_form, post)
@@ -346,7 +346,7 @@ class TestSchemaNormalization(unittest.TestCase):
'ring"]},{"name":"serverHash","type":["null",{"name":"org.ap'
'ache.avro.ipc.MD5","type":"fixed","size":16}]},{"name":"met'
'a","type":["null",{"type":"map","values":"bytes"}]}]}')
- pre_schema = Parse(pre)
+ pre_schema = parse(pre)
canonical_form = ToParsingCanonicalForm(pre_schema)
self.assertEqual(canonical_form, post)
@@ -415,7 +415,7 @@ class TestSchemaNormalization(unittest.TestCase):
'"recordField","type":{"name":"org.apache.avro.Node","type":"reco'
'rd","fields":[{"name":"label","type":"string"},{"name":"children'
'","type":{"type":"array","items":"org.apache.avro.Node"}}]}}]}')
- pre_schema = Parse(pre)
+ pre_schema = parse(pre)
canonical_form = ToParsingCanonicalForm(pre_schema)
self.assertEqual(canonical_form, post)
@@ -436,7 +436,7 @@ class TestSchemaNormalization(unittest.TestCase):
post=('{"name":"ipAddr","type":"record","fields":[{"name"'
':"addr","type":[{"name":"IPv6","type":"fixed","siz'
'e":16},{"name":"IPv4","type":"fixed","size":4}]}]}')
- pre_schema = Parse(pre)
+ pre_schema = parse(pre)
canonical_form = ToParsingCanonicalForm(pre_schema)
self.assertEqual(canonical_form, post)
@@ -451,7 +451,7 @@ class TestSchemaNormalization(unittest.TestCase):
"""
post=('{"name":"TestDoc","type":"record","fiel'
'ds":[{"name":"name","type":"string"}]}')
- pre_schema = Parse(pre)
+ pre_schema = parse(pre)
canonical_form = ToParsingCanonicalForm(pre_schema)
self.assertEqual(canonical_form, post)
@@ -460,7 +460,7 @@ class TestSchemaNormalization(unittest.TestCase):
{"type": "enum", "name": "Test", "symbols": ["A", "B"], "doc": "Doc String"}
"""
post='{"name":"Test","type":"enum","symbols":["A","B"]}'
- pre_schema = Parse(pre)
+ pre_schema = parse(pre)
canonical_form = ToParsingCanonicalForm(pre_schema)
self.assertEqual(canonical_form, post)
@@ -487,7 +487,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
def testUnsupportedFingerprintAlgorithmRaisesValueError(self):
schema='"int"'
algorithm='UNKNOWN'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
with self.assertRaises(ValueError) as context:
Fingerprint(normal_form, algorithm)
@@ -497,7 +497,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='"int"'
algorithm='CRC-64-AVRO'
expected_hex_fingerprint='8f5c393f1ad57572'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
@@ -507,7 +507,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='"int"'
algorithm='md5'
expected_hex_fingerprint='ef524ea1b91e73173d938ade36c1db32'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
@@ -517,7 +517,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='"int"'
algorithm='sha256'
expected_hex_fingerprint='3f2b87a9fe7cc9b13835598c3981cd45e3e355309e5090aa0933d7becb6fba45'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
@@ -527,7 +527,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='{"type": "int"}'
algorithm='CRC-64-AVRO'
expected_hex_fingerprint='8f5c393f1ad57572'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
@@ -537,7 +537,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='{"type": "int"}'
algorithm='md5'
expected_hex_fingerprint='ef524ea1b91e73173d938ade36c1db32'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
@@ -547,7 +547,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='{"type": "int"}'
algorithm='sha256'
expected_hex_fingerprint='3f2b87a9fe7cc9b13835598c3981cd45e3e355309e5090aa0933d7becb6fba45'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
@@ -557,7 +557,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='"float"'
algorithm='CRC-64-AVRO'
expected_hex_fingerprint='90d7a83ecb027c4d'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
@@ -567,7 +567,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='"float"'
algorithm='md5'
expected_hex_fingerprint='50a6b9db85da367a6d2df400a41758a6'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
@@ -577,7 +577,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='"float"'
algorithm='sha256'
expected_hex_fingerprint='1e71f9ec051d663f56b0d8e1fc84d71aa56ccfe9fa93aa20d10547a7abeb5cc0'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
@@ -587,7 +587,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='{"type": "float"}'
algorithm='CRC-64-AVRO'
expected_hex_fingerprint='90d7a83ecb027c4d'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
@@ -597,7 +597,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='{"type": "float"}'
algorithm='md5'
expected_hex_fingerprint='50a6b9db85da367a6d2df400a41758a6'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
@@ -607,7 +607,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='{"type": "float"}'
algorithm='sha256'
expected_hex_fingerprint='1e71f9ec051d663f56b0d8e1fc84d71aa56ccfe9fa93aa20d10547a7abeb5cc0'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
@@ -617,7 +617,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='"long"'
algorithm='CRC-64-AVRO'
expected_hex_fingerprint='b71df49344e154d0'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
@@ -627,7 +627,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='"long"'
algorithm='md5'
expected_hex_fingerprint='e1dd9a1ef98b451b53690370b393966b'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
@@ -637,7 +637,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='"long"'
algorithm='sha256'
expected_hex_fingerprint='c32c497df6730c97fa07362aa5023f37d49a027ec452360778114cf427965add'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
@@ -647,7 +647,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='{"type": "long"}'
algorithm='CRC-64-AVRO'
expected_hex_fingerprint='b71df49344e154d0'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
@@ -657,7 +657,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='{"type": "long"}'
algorithm='md5'
expected_hex_fingerprint='e1dd9a1ef98b451b53690370b393966b'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
@@ -667,7 +667,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='{"type": "long"}'
algorithm='sha256'
expected_hex_fingerprint='c32c497df6730c97fa07362aa5023f37d49a027ec452360778114cf427965add'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
@@ -677,7 +677,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='"double"'
algorithm='CRC-64-AVRO'
expected_hex_fingerprint='7e95ab32c035758e'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
@@ -687,7 +687,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='"double"'
algorithm='md5'
expected_hex_fingerprint='bfc71a62f38b99d6a93690deeb4b3af6'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
@@ -697,7 +697,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='"double"'
algorithm='sha256'
expected_hex_fingerprint='730a9a8c611681d7eef442e03c16c70d13bca3eb8b977bb403eaff52176af254'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
@@ -707,7 +707,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='{"type": "double"}'
algorithm='CRC-64-AVRO'
expected_hex_fingerprint='7e95ab32c035758e'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
@@ -717,7 +717,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='{"type": "double"}'
algorithm='md5'
expected_hex_fingerprint='bfc71a62f38b99d6a93690deeb4b3af6'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
@@ -727,7 +727,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='{"type": "double"}'
algorithm='sha256'
expected_hex_fingerprint='730a9a8c611681d7eef442e03c16c70d13bca3eb8b977bb403eaff52176af254'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
@@ -737,7 +737,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='"bytes"'
algorithm='CRC-64-AVRO'
expected_hex_fingerprint='651920c3da16c04f'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
@@ -747,7 +747,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='"bytes"'
algorithm='md5'
expected_hex_fingerprint='b462f06cb909be57c85008867784cde6'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
@@ -757,7 +757,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='"bytes"'
algorithm='sha256'
expected_hex_fingerprint='9ae507a9dd39ee5b7c7e285da2c0846521c8ae8d80feeae5504e0c981d53f5fa'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
@@ -767,7 +767,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='{"type": "bytes"}'
algorithm='CRC-64-AVRO'
expected_hex_fingerprint='651920c3da16c04f'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
@@ -777,7 +777,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='{"type": "bytes"}'
algorithm='md5'
expected_hex_fingerprint='b462f06cb909be57c85008867784cde6'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
@@ -787,7 +787,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='{"type": "bytes"}'
algorithm='sha256'
expected_hex_fingerprint='9ae507a9dd39ee5b7c7e285da2c0846521c8ae8d80feeae5504e0c981d53f5fa'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
@@ -797,7 +797,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='"string"'
algorithm='CRC-64-AVRO'
expected_hex_fingerprint='c70345637248018f'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
@@ -807,7 +807,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='"string"'
algorithm='md5'
expected_hex_fingerprint='095d71cf12556b9d5e330ad575b3df5d'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
@@ -817,7 +817,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='"string"'
algorithm='sha256'
expected_hex_fingerprint='e9e5c1c9e4f6277339d1bcde0733a59bd42f8731f449da6dc13010a916930d48'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
@@ -827,7 +827,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='{"type": "string"}'
algorithm='CRC-64-AVRO'
expected_hex_fingerprint='c70345637248018f'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
@@ -837,7 +837,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='{"type": "string"}'
algorithm='md5'
expected_hex_fingerprint='095d71cf12556b9d5e330ad575b3df5d'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
@@ -847,7 +847,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='{"type": "string"}'
algorithm='sha256'
expected_hex_fingerprint='e9e5c1c9e4f6277339d1bcde0733a59bd42f8731f449da6dc13010a916930d48'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
@@ -857,7 +857,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='"boolean"'
algorithm='CRC-64-AVRO'
expected_hex_fingerprint='64f7d4a478fc429f'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
@@ -867,7 +867,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='"boolean"'
algorithm='md5'
expected_hex_fingerprint='01f692b30d4a1c8a3e600b1440637f8f'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
@@ -877,7 +877,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='"boolean"'
algorithm='sha256'
expected_hex_fingerprint='a5b031ab62bc416d720c0410d802ea46b910c4fbe85c50a946ccc658b74e677e'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
@@ -887,7 +887,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='{"type": "boolean"}'
algorithm='CRC-64-AVRO'
expected_hex_fingerprint='64f7d4a478fc429f'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
@@ -897,7 +897,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='{"type": "boolean"}'
algorithm='md5'
expected_hex_fingerprint='01f692b30d4a1c8a3e600b1440637f8f'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
@@ -907,7 +907,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='{"type": "boolean"}'
algorithm='sha256'
expected_hex_fingerprint='a5b031ab62bc416d720c0410d802ea46b910c4fbe85c50a946ccc658b74e677e'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
@@ -917,7 +917,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='"null"'
algorithm='CRC-64-AVRO'
expected_hex_fingerprint='8a8f25cce724dd63'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
@@ -927,7 +927,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='"null"'
algorithm='md5'
expected_hex_fingerprint='9b41ef67651c18488a8b08bb67c75699'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
@@ -937,7 +937,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='"null"'
algorithm='sha256'
expected_hex_fingerprint='f072cbec3bf8841871d4284230c5e983dc211a56837aed862487148f947d1a1f'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
@@ -947,7 +947,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='{"type": "null"}'
algorithm='CRC-64-AVRO'
expected_hex_fingerprint='8a8f25cce724dd63'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
@@ -957,7 +957,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='{"type": "null"}'
algorithm='md5'
expected_hex_fingerprint='9b41ef67651c18488a8b08bb67c75699'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
@@ -967,7 +967,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='{"type": "null"}'
algorithm='sha256'
expected_hex_fingerprint='f072cbec3bf8841871d4284230c5e983dc211a56837aed862487148f947d1a1f'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
@@ -977,7 +977,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='{"type": "fixed", "name": "Test", "size": 1}'
algorithm='CRC-64-AVRO'
expected_hex_fingerprint='6869897b4049355b'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
@@ -987,7 +987,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='{"type": "fixed", "name": "Test", "size": 1}'
algorithm='md5'
expected_hex_fingerprint='db01bc515fcfcd2d4be82ed385288261'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
@@ -997,7 +997,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='{"type": "fixed", "name": "Test", "size": 1}'
algorithm='sha256'
expected_hex_fingerprint='f527116a6f44455697e935afc31dc60ad0f95caf35e1d9c9db62edb3ffeb9170'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
@@ -1014,7 +1014,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
"""
algorithm='CRC-64-AVRO'
expected_hex_fingerprint='fadbd138e85bdf45'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
@@ -1031,7 +1031,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
"""
algorithm='md5'
expected_hex_fingerprint='d74b3726484422711c465d49e857b1ba'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
@@ -1048,7 +1048,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
"""
algorithm='sha256'
expected_hex_fingerprint='28e493a44771cecc5deca4bd938cdc3d5a24cfe1f3760bc938fa1057df6334fc'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
@@ -1058,7 +1058,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='{"type": "enum", "name": "Test", "symbols": ["A", "B"]}'
algorithm='CRC-64-AVRO'
expected_hex_fingerprint='03a2f2c2e27f7a16'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
@@ -1068,7 +1068,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='{"type": "enum", "name": "Test", "symbols": ["A", "B"]}'
algorithm='md5'
expected_hex_fingerprint='d883f2a9b16ed085fcc5e4ca6c8f6ed1'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
@@ -1078,7 +1078,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='{"type": "enum", "name": "Test", "symbols": ["A", "B"]}'
algorithm='sha256'
expected_hex_fingerprint='9b51286144f87ce5aebdc61ca834379effa5a41ce6ac0938630ff246297caca8'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
@@ -1088,7 +1088,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='{"type": "array", "items": "long"}'
algorithm='CRC-64-AVRO'
expected_hex_fingerprint='715e2ea28bc91654'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
@@ -1098,7 +1098,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='{"type": "array", "items": "long"}'
algorithm='md5'
expected_hex_fingerprint='c1c387e8d6a58f0df749b698991b1f43'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
@@ -1108,7 +1108,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='{"type": "array", "items": "long"}'
algorithm='sha256'
expected_hex_fingerprint='f78e954167feb23dcb1ce01e8463cebf3408e0a4259e16f24bd38f6d0f1d578b'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
@@ -1123,7 +1123,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
"""
algorithm='CRC-64-AVRO'
expected_hex_fingerprint='10d9ade1fa3a0387'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
@@ -1138,7 +1138,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
"""
algorithm='md5'
expected_hex_fingerprint='cfc7b861c7cfef082a6ef082948893fa'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
@@ -1153,7 +1153,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
"""
algorithm='sha256'
expected_hex_fingerprint='0d8edd49d7f7e9553668f133577bc99f842852b55d9f84f1f7511e4961aa685c'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
@@ -1163,7 +1163,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='{"type": "map", "values": "long"}'
algorithm='CRC-64-AVRO'
expected_hex_fingerprint='6f74f4e409b1334e'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
@@ -1173,7 +1173,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='{"type": "map", "values": "long"}'
algorithm='md5'
expected_hex_fingerprint='32b3f1a3177a0e73017920f00448b56e'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
@@ -1183,7 +1183,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='{"type": "map", "values": "long"}'
algorithm='sha256'
expected_hex_fingerprint='b8fad07d458971a07692206b8a7cf626c86c62fe6bcff7c1b11bc7295de34853'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
@@ -1198,7 +1198,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
"""
algorithm='CRC-64-AVRO'
expected_hex_fingerprint='df2ab0626f6b812d'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
@@ -1213,7 +1213,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
"""
algorithm='md5'
expected_hex_fingerprint='c588da6ba99701c41e73fd30d23f994e'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
@@ -1228,7 +1228,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
"""
algorithm='sha256'
expected_hex_fingerprint='3886747ed1669a8af476b549e97b34222afb2fed5f18bb27c6f367ea0351a576'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
@@ -1238,7 +1238,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='["string", "null", "long"]'
algorithm='CRC-64-AVRO'
expected_hex_fingerprint='65a5be410d687566'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
@@ -1248,7 +1248,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='["string", "null", "long"]'
algorithm='md5'
expected_hex_fingerprint='b11cf95f0a55dd55f9ee515a37bf937a'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
@@ -1258,7 +1258,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='["string", "null", "long"]'
algorithm='sha256'
expected_hex_fingerprint='ed8d254116441bb35e237ad0563cf5432b8c975334bd222c1ee84609435d95bb'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
@@ -1274,7 +1274,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
"""
algorithm='CRC-64-AVRO'
expected_hex_fingerprint='ed94e5f5e6eb588e'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
@@ -1290,7 +1290,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
"""
algorithm='md5'
expected_hex_fingerprint='69531a03db788afe353244cd049b1e6d'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
@@ -1306,7 +1306,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
"""
algorithm='sha256'
expected_hex_fingerprint='9670f15a8f96d23e92830d00b8bd57275e02e3e173ffef7c253c170b6beabeb8'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
@@ -1322,7 +1322,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
"""
algorithm='CRC-64-AVRO'
expected_hex_fingerprint='ed94e5f5e6eb588e'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
@@ -1338,7 +1338,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
"""
algorithm='md5'
expected_hex_fingerprint='69531a03db788afe353244cd049b1e6d'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
@@ -1354,7 +1354,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
"""
algorithm='sha256'
expected_hex_fingerprint='9670f15a8f96d23e92830d00b8bd57275e02e3e173ffef7c253c170b6beabeb8'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
@@ -1373,7 +1373,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
"""
algorithm='CRC-64-AVRO'
expected_hex_fingerprint='52cba544c3e756b7'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
@@ -1392,7 +1392,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
"""
algorithm='md5'
expected_hex_fingerprint='99625b0cc02050363e89ef66b0f406c9'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
@@ -1411,7 +1411,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
"""
algorithm='sha256'
expected_hex_fingerprint='65d80dc8c95c98a9671d92cf0415edfabfee2cb058df2138606656cd6ae4dc59'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
@@ -1439,7 +1439,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
"""
algorithm='CRC-64-AVRO'
expected_hex_fingerprint='68d91a23eda0b306'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
@@ -1467,7 +1467,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
"""
algorithm='md5'
expected_hex_fingerprint='9e1d0d15b52789fcb8e3a88b53059d5f'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
@@ -1495,7 +1495,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
"""
algorithm='sha256'
expected_hex_fingerprint='e5ce4f4a15ce19fa1047cfe16a3b0e13a755db40f00f23284fdd376fc1c7dd21'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
@@ -1523,7 +1523,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
"""
algorithm='CRC-64-AVRO'
expected_hex_fingerprint='b96ad79e5a7c5757'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
@@ -1551,7 +1551,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
"""
algorithm='md5'
expected_hex_fingerprint='4c822af2e17eecd92422827eede97f5b'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
@@ -1579,7 +1579,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
"""
algorithm='sha256'
expected_hex_fingerprint='2b2f7a9b22991fe0df9134cb6b5ff7355343e797aaea337e0150e20f3a35800e'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
@@ -1612,7 +1612,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
"""
algorithm='CRC-64-AVRO'
expected_hex_fingerprint='00feee01de4ea50e'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
@@ -1645,7 +1645,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
"""
algorithm='md5'
expected_hex_fingerprint='afe529d01132daab7f4e2a6663e7a2f5'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
@@ -1678,7 +1678,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
"""
algorithm='sha256'
expected_hex_fingerprint='a303cbbfe13958f880605d70c521a4b7be34d9265ac5a848f25916a67b11d889'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
@@ -1735,7 +1735,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
"""
algorithm='CRC-64-AVRO'
expected_hex_fingerprint='e82c0a93a6a0b5a4'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
@@ -1792,7 +1792,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
"""
algorithm='md5'
expected_hex_fingerprint='994fea1a1be7ff8603cbe40c3bc7e4ca'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
@@ -1849,7 +1849,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
"""
algorithm='sha256'
expected_hex_fingerprint='cccfd6e3f917cf53b0f90c206342e6703b0d905071f724a1c1f85b731c74058d'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
@@ -1871,7 +1871,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
"""
algorithm='CRC-64-AVRO'
expected_hex_fingerprint='8d961b4e298a1844'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
@@ -1893,7 +1893,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
"""
algorithm='md5'
expected_hex_fingerprint='45d85c69b353a99b93d7c4f2fcf0c30d'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
@@ -1915,7 +1915,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
"""
algorithm='sha256'
expected_hex_fingerprint='6f6fc8f685a4f07d99734946565d63108806d55a8620febea047cf52cb0ac181'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
@@ -1932,7 +1932,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
"""
algorithm='CRC-64-AVRO'
expected_hex_fingerprint='0e6660f02bcdc109'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
@@ -1949,7 +1949,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
"""
algorithm='md5'
expected_hex_fingerprint='f2da75f5131f5ab80629538287b8beb2'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
@@ -1966,7 +1966,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
"""
algorithm='sha256'
expected_hex_fingerprint='0b3644f7aa5ca2fc4bad93ca2d3609c12aa9dbda9c15e68b34c120beff08e7b9'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
@@ -1978,7 +1978,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
"""
algorithm='CRC-64-AVRO'
expected_hex_fingerprint='03a2f2c2e27f7a16'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(16)
@@ -1990,7 +1990,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
"""
algorithm='md5'
expected_hex_fingerprint='d883f2a9b16ed085fcc5e4ca6c8f6ed1'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
@@ -2002,7 +2002,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
"""
algorithm='sha256'
expected_hex_fingerprint='9b51286144f87ce5aebdc61ca834379effa5a41ce6ac0938630ff246297caca8'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
@@ -2012,7 +2012,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='{"type": "int"}'
algorithm='MD5' # Java compatible name alias
expected_hex_fingerprint='ef524ea1b91e73173d938ade36c1db32'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(32)
@@ -2022,7 +2022,7 @@ class TestSchemaFingerprinting(unittest.TestCase):
schema='{"type": "int"}'
algorithm='SHA-256' # Java compatible name alias
expected_hex_fingerprint='3f2b87a9fe7cc9b13835598c3981cd45e3e355309e5090aa0933d7becb6fba45'
- pre_schema = Parse(schema)
+ pre_schema = parse(schema)
normal_form = ToParsingCanonicalForm(pre_schema)
fingerprint = Fingerprint(normal_form, algorithm)
actual_hex_fingerprint = "".join(format(b, "02x") for b in fingerprint).zfill(64)
diff --git a/lang/py3/avro/tests/test_protocol.py b/lang/py3/avro/tests/test_protocol.py
index 7dabd59..d372973 100644
--- a/lang/py3/avro/tests/test_protocol.py
+++ b/lang/py3/avro/tests/test_protocol.py
@@ -24,6 +24,7 @@ Test the protocol parsing logic.
import logging
import traceback
import unittest
+import unittest.mock
from avro import protocol
@@ -399,6 +400,12 @@ VALID_EXAMPLES = [e for e in EXAMPLES if e.valid]
class TestProtocol(unittest.TestCase):
+ @unittest.mock.patch('avro.protocol.warnings.warn')
+ def test_Parse_is_deprecated(self, warn):
+ """Capital-P Parse is deprecated."""
+ protocol.Parse(HELLO_WORLD.protocol_string)
+ self.assertEqual(warn.call_count, 1)
+
def testParse(self):
correct = 0
for iexample, example in enumerate(EXAMPLES):
@@ -406,7 +413,7 @@ class TestProtocol(unittest.TestCase):
'Parsing protocol #%d:\n%s',
iexample, example.protocol_string)
try:
- parsed = protocol.Parse(example.protocol_string)
+ parsed = protocol.parse(example.protocol_string)
if example.valid:
correct += 1
else:
@@ -431,13 +438,13 @@ class TestProtocol(unittest.TestCase):
% (correct, len(EXAMPLES)))
def testInnerNamespaceSet(self):
- proto = protocol.Parse(HELLO_WORLD.protocol_string)
+ proto = protocol.parse(HELLO_WORLD.protocol_string)
self.assertEqual(proto.namespace, 'com.acme')
greeting_type = proto.type_map['com.acme.Greeting']
self.assertEqual(greeting_type.namespace, 'com.acme')
def testInnerNamespaceNotRendered(self):
- proto = protocol.Parse(HELLO_WORLD.protocol_string)
+ proto = protocol.parse(HELLO_WORLD.protocol_string)
self.assertEqual('com.acme.Greeting', proto.types[0].fullname)
self.assertEqual('Greeting', proto.types[0].name)
# but there shouldn't be 'namespace' rendered to json on the inner type
@@ -450,9 +457,9 @@ class TestProtocol(unittest.TestCase):
"""
num_correct = 0
for example in VALID_EXAMPLES:
- proto = protocol.Parse(example.protocol_string)
+ proto = protocol.parse(example.protocol_string)
try:
- protocol.Parse(str(proto))
+ protocol.parse(str(proto))
except ProtocolParseException:
logging.debug('Failed to reparse protocol:\n%s',
example.protocol_string)
@@ -460,7 +467,6 @@ class TestProtocol(unittest.TestCase):
logging.debug('Successfully reparsed protocol:\n%s',
example.protocol_string)
num_correct += 1
-
fail_msg = (
'Cast to string success on %d out of %d protocols'
% (num_correct, len(VALID_EXAMPLES)))
@@ -475,8 +481,8 @@ class TestProtocol(unittest.TestCase):
"""
num_correct = 0
for example in VALID_EXAMPLES:
- original_protocol = protocol.Parse(example.protocol_string)
- round_trip_protocol = protocol.Parse(str(original_protocol))
+ original_protocol = protocol.parse(example.protocol_string)
+ round_trip_protocol = protocol.parse(str(original_protocol))
if original_protocol == round_trip_protocol:
num_correct += 1
diff --git a/lang/py3/avro/tests/test_schema.py b/lang/py3/avro/tests/test_schema.py
index 2d82904..dc16513 100644
--- a/lang/py3/avro/tests/test_schema.py
+++ b/lang/py3/avro/tests/test_schema.py
@@ -24,6 +24,7 @@ Test the schema parsing logic.
import logging
import traceback
import unittest
+import unittest.mock
from avro import schema
@@ -450,7 +451,7 @@ VALID_EXAMPLES = [e for e in EXAMPLES if e.valid]
class TestSchema(unittest.TestCase):
def testCorrectRecursiveExtraction(self):
- parsed = schema.Parse("""
+ parsed = schema.parse("""
{
"type": "record",
"name": "X",
@@ -466,7 +467,7 @@ class TestSchema(unittest.TestCase):
""")
logging.debug('Parsed schema:\n%s', parsed)
logging.debug('Fields: %s', parsed.fields)
- t = schema.Parse(str(parsed.fields[0].type))
+ t = schema.parse(str(parsed.fields[0].type))
# If we've made it this far, the subschema was reasonably stringified;
# it could be reparsed.
self.assertEqual("X", t.fields[0].type.name)
@@ -480,16 +481,22 @@ class TestSchema(unittest.TestCase):
'{"type": "map", "values": "int"}',
'["null", "int"]',
]
- for case in (schema.Parse(case) for case in cases):
+ for case in (schema.parse(case) for case in cases):
self.assertRaises(AttributeError, lambda: case.name)
self.assertEqual(getattr(case, "name", "default"), "default")
- def testParse(self):
+ @unittest.mock.patch('avro.schema.warnings.warn')
+ def test_Parse_is_deprecated(self, warn):
+ """Capital-P Parse is deprecated."""
+ schema.Parse(PRIMITIVE_EXAMPLES[0].schema_string)
+ self.assertEqual(warn.call_count, 1)
+
+ def test_parse(self):
correct = 0
for iexample, example in enumerate(EXAMPLES):
logging.debug('Testing example #%d\n%s', iexample, example.schema_string)
try:
- schema.Parse(example.schema_string)
+ schema.parse(example.schema_string)
if example.valid:
correct += 1
else:
@@ -520,8 +527,8 @@ class TestSchema(unittest.TestCase):
"""
correct = 0
for example in VALID_EXAMPLES:
- schema_data = schema.Parse(example.schema_string)
- schema.Parse(str(schema_data))
+ schema_data = schema.parse(example.schema_string)
+ schema.parse(str(schema_data))
correct += 1
fail_msg = "Cast to string success on %d out of %d schemas" % \
@@ -537,8 +544,8 @@ class TestSchema(unittest.TestCase):
"""
correct = 0
for example in VALID_EXAMPLES:
- original_schema = schema.Parse(example.schema_string)
- round_trip_schema = schema.Parse(str(original_schema))
+ original_schema = schema.parse(example.schema_string)
+ round_trip_schema = schema.parse(str(original_schema))
if original_schema == round_trip_schema:
correct += 1
debug_msg = "%s: ROUND TRIP SUCCESS" % example.name
@@ -592,7 +599,7 @@ class TestSchema(unittest.TestCase):
def testDocAttributes(self):
correct = 0
for example in DOC_EXAMPLES:
- original_schema = schema.Parse(example.schema_string)
+ original_schema = schema.parse(example.schema_string)
if original_schema.doc is not None:
correct += 1
if original_schema.type == 'record':
@@ -607,8 +614,8 @@ class TestSchema(unittest.TestCase):
correct = 0
props = {}
for example in OTHER_PROP_EXAMPLES:
- original_schema = schema.Parse(example.schema_string)
- round_trip_schema = schema.Parse(str(original_schema))
+ original_schema = schema.parse(example.schema_string)
+ round_trip_schema = schema.parse(str(original_schema))
self.assertEqual(original_schema.other_props,round_trip_schema.other_props)
if original_schema.type == "record":
field_props = 0
@@ -641,8 +648,8 @@ class TestSchema(unittest.TestCase):
"fields": [{"name": "foo", "type": "int"}, {"name": "foo", "type": "string"}]
}"""
with self.assertRaises(schema.SchemaParseException) as e:
- schema.Parse(schema_string)
- self.assertRegexpMatches(str(e.exception), 'Duplicate.*field name.*foo')
+ schema.parse(schema_string)
+ self.assertRegex(str(e.exception), 'Duplicate.*field name.*foo')
# ------------------------------------------------------------------------------
diff --git a/lang/py3/avro/tests/test_script.py b/lang/py3/avro/tests/test_script.py
index 647d346..4d317dd 100644
--- a/lang/py3/avro/tests/test_script.py
+++ b/lang/py3/avro/tests/test_script.py
@@ -118,7 +118,7 @@ class TestCat(unittest.TestCase):
@staticmethod
def WriteAvroFile(file_path):
- schema = avro.schema.Parse(SCHEMA)
+ schema = avro.schema.parse(SCHEMA)
with open(file_path, 'wb') as writer:
with avro.datafile.DataFileWriter(
writer=writer,
diff --git a/lang/py3/avro/tests/txsample_http_client.py b/lang/py3/avro/tests/txsample_http_client.py
index f3e573a..a536aa8 100644
--- a/lang/py3/avro/tests/txsample_http_client.py
+++ b/lang/py3/avro/tests/txsample_http_client.py
@@ -49,7 +49,7 @@ MAIL_PROTOCOL_JSON = """\
}
}
"""
-MAIL_PROTOCOL = protocol.Parse(MAIL_PROTOCOL_JSON)
+MAIL_PROTOCOL = protocol.parse(MAIL_PROTOCOL_JSON)
SERVER_HOST = 'localhost'
SERVER_PORT = 9090
diff --git a/lang/py3/avro/tests/txsample_http_server.py b/lang/py3/avro/tests/txsample_http_server.py
index c85f42a..beb1267 100644
--- a/lang/py3/avro/tests/txsample_http_server.py
+++ b/lang/py3/avro/tests/txsample_http_server.py
@@ -49,7 +49,7 @@ MAIL_PROTOCOL_JSON = """\
}
}
"""
-MAIL_PROTOCOL = protocol.Parse(MAIL_PROTOCOL_JSON)
+MAIL_PROTOCOL = protocol.parse(MAIL_PROTOCOL_JSON)
SERVER_ADDRESS = ('localhost', 9090)
class MailResponder(ipc.Responder):
diff --git a/lang/py3/avro/tool.py b/lang/py3/avro/tool.py
index 6bfce0d..e3674b2 100644
--- a/lang/py3/avro/tool.py
+++ b/lang/py3/avro/tool.py
@@ -25,21 +25,21 @@ NOTE: The API for the command-line tool is experimental.
import sys
import urllib
-from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
+from http.server import BaseHTTPRequestHandler, HTTPServer
from avro import datafile, io, ipc, protocol
class GenericResponder(ipc.Responder):
def __init__(self, proto, msg, datum):
- proto_json = file(proto, 'r').read()
- ipc.Responder.__init__(self, protocol.Parse(proto_json))
+ proto_json = open(proto, 'r').read()
+ ipc.Responder.__init__(self, protocol.parse(proto_json))
self.msg = msg
self.datum = datum
- def invoke(self, message, request):
+ def Invoke(self, message, request):
if message.name == self.msg:
- print >> sys.stderr, "Message: %s Datum: %s" % (message.name, self.datum)
+ print("Message: %s Datum: %s" % (message.name, self.datum), sys.stderr)
# server will shut down after processing a single Avro request
global server_should_shutdown
server_should_shutdown = True
@@ -49,15 +49,15 @@ class GenericHandler(BaseHTTPRequestHandler):
def do_POST(self):
self.responder = responder
call_request_reader = ipc.FramedReader(self.rfile)
- call_request = call_request_reader.read_framed_message()
- resp_body = self.responder.respond(call_request)
+ call_request = call_request_reader.Read()
+ resp_body = self.responder.Respond(call_request)
self.send_response(200)
self.send_header('Content-Type', 'avro/binary')
self.end_headers()
resp_writer = ipc.FramedWriter(self.wfile)
- resp_writer.write_framed_message(resp_body)
+ resp_writer.Write(resp_body)
if server_should_shutdown:
- print >> sys.stderr, "Shutting down server."
+ print("Shutting down server.", sys.stderr)
self.server.force_stop()
class StoppableHTTPServer(HTTPServer):
@@ -88,15 +88,15 @@ def run_server(uri, proto, msg, datum):
print("Port: %s" % server.server_port)
sys.stdout.flush()
server.allow_reuse_address = True
- print >> sys.stderr, "Starting server."
+ print("Starting server.", sys.stderr)
server.serve_forever()
def send_message(uri, proto, msg, datum):
url_obj = urllib.parse.urlparse(uri)
client = ipc.HTTPTransceiver(url_obj.hostname, url_obj.port)
- proto_json = file(proto, 'r').read()
- requestor = ipc.Requestor(protocol.Parse(proto_json), client)
- print(requestor.request(msg, datum))
+ proto_json = open(proto, 'r').read()
+ requestor = ipc.Requestor(protocol.parse(proto_json), client)
+ print(requestor.Request(msg, datum))
def file_or_stdin(f):
if f == "-":
@@ -125,10 +125,9 @@ def main(args=sys.argv):
datum = None
if len(args) > 5:
if args[5] == "-file":
- reader = open(args[6], 'rb')
- datum_reader = io.DatumReader()
- dfr = datafile.DataFileReader(reader, datum_reader)
- datum = dfr.next()
+ with open(args[6], 'rb') as reader:
+ with datafile.DataFileReader(reader, io.DatumReader()) as dfr:
+ datum = next(dfr)
elif args[5] == "-data":
print("JSON Decoder not yet implemented.")
return 1
@@ -146,10 +145,9 @@ def main(args=sys.argv):
datum = None
if len(args) > 5:
if args[5] == "-file":
- reader = open(args[6], 'rb')
- datum_reader = io.DatumReader()
- dfr = datafile.DataFileReader(reader, datum_reader)
- datum = dfr.next()
+ with open(args[6], 'rb') as reader:
+ with datafile.DataFileReader(reader, io.DatumReader()) as dfr:
+ datum = next(dfr)
elif args[5] == "-data":
print("JSON Decoder not yet implemented.")
return 1
diff --git a/lang/py3/scripts/avro b/lang/py3/scripts/avro
index 61d6a4f..23daac8 100755
--- a/lang/py3/scripts/avro
+++ b/lang/py3/scripts/avro
@@ -213,7 +213,7 @@ def write(opts, files):
try:
with open(opts.schema, 'rt') as f:
json_schema = f.read()
- writer_schema = schema.Parse(json_schema)
+ writer_schema = schema.parse(json_schema)
out = _open(opts.output, 'wb')
except (IOError, OSError) as e:
raise AvroError('Cannot open file - %s' % e)