You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by fo...@apache.org on 2019/07/16 13:50:05 UTC
[avro] branch master updated: AVRO-2469: Add data interop test to
the Python3 bindings (#581)
This is an automated email from the ASF dual-hosted git repository.
fokko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/avro.git
The following commit(s) were added to refs/heads/master by this push:
new fcb4764 AVRO-2469: Add data interop test to the Python3 bindings (#581)
fcb4764 is described below
commit fcb4764468cd1d70b3341c1488a394bb8f20929b
Author: Kengo Seki <se...@apache.org>
AuthorDate: Tue Jul 16 22:49:58 2019 +0900
AVRO-2469: Add data interop test to the Python3 bindings (#581)
* AVRO-2469: Add data interop test to the Python3 bindings
* Introduce with statement and pathlib to make the syntax clean
* Use DataFileReader and DataFileWriter with "with" statements
---
build.sh | 3 ++
lang/py3/avro/tests/gen_interop_data.py | 21 ++++++----
lang/py3/avro/tests/test_datafile_interop.py | 61 +++++-----------------------
lang/py3/setup.py | 28 ++++++++++++-
4 files changed, 54 insertions(+), 59 deletions(-)
diff --git a/build.sh b/build.sh
index e608946..d5cf479 100755
--- a/build.sh
+++ b/build.sh
@@ -61,6 +61,8 @@ do
(cd lang/perl; ./build.sh test)
(cd lang/py; ant interop-data-generate)
+ (cd lang/py3; python3 setup.py generate_interop_data \
+ --schema-file=../../share/test/schemas/interop.avsc --output-path=../../build/interop/data)
(cd lang/c; ./build.sh interop-data-generate)
#(cd lang/c++; make interop-data-generate)
(cd lang/csharp; ./build.sh interop-data-generate)
@@ -70,6 +72,7 @@ do
# run interop data tests
(cd lang/java; mvn -B test -P interop-data-test)
(cd lang/py; ant interop-data-test)
+ (cd lang/py3; python3 setup.py test --test-suite avro.tests.test_datafile_interop.TestDataFileInterop)
(cd lang/c; ./build.sh interop-data-test)
#(cd lang/c++; make interop-data-test)
(cd lang/csharp; ./build.sh interop-data-test)
diff --git a/lang/py3/avro/tests/gen_interop_data.py b/lang/py3/avro/tests/gen_interop_data.py
index 2180b2d..723dca5 100644
--- a/lang/py3/avro/tests/gen_interop_data.py
+++ b/lang/py3/avro/tests/gen_interop_data.py
@@ -19,6 +19,7 @@
# limitations under the License.
import sys
+from pathlib import Path
from avro import datafile
from avro import io
@@ -46,11 +47,17 @@ DATUM = {
}
-if __name__ == "__main__":
- interop_schema = schema.Parse(open(sys.argv[1], 'r').read())
- writer = open(sys.argv[2], 'wb')
+def generate(schema_file, output_path):
+ interop_schema = schema.Parse(open(schema_file, 'r').read())
datum_writer = io.DatumWriter()
- # NB: not using compression
- dfw = datafile.DataFileWriter(writer, datum_writer, interop_schema)
- dfw.append(DATUM)
- dfw.close()
+ for codec in datafile.VALID_CODECS:
+ filename = 'py3'
+ if codec != 'null':
+ filename += '_' + codec
+ with Path(output_path, filename).with_suffix('.avro').open('wb') as writer, \
+ datafile.DataFileWriter(writer, datum_writer, interop_schema, codec) as dfw:
+ dfw.append(DATUM)
+
+
+if __name__ == "__main__":
+ generate(sys.argv[1], sys.argv[2])
diff --git a/lang/py3/avro/tests/test_datafile_interop.py b/lang/py3/avro/tests/test_datafile_interop.py
index df2ce68..12d5605 100644
--- a/lang/py3/avro/tests/test_datafile_interop.py
+++ b/lang/py3/avro/tests/test_datafile_interop.py
@@ -18,65 +18,26 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import logging
+import glob
import os
-import tempfile
import unittest
from avro import datafile
from avro import io
-from avro import schema
-
-
-def GetInteropSchema():
- test_dir = os.path.dirname(os.path.abspath(__file__))
- schema_json_path = os.path.join(test_dir, 'interop.avsc')
- with open(schema_json_path, 'r') as f:
- schema_json = f.read()
- return schema.Parse(schema_json)
-
-
-INTEROP_SCHEMA = GetInteropSchema()
-INTEROP_DATUM = {
- 'intField': 12,
- 'longField': 15234324,
- 'stringField': 'hey',
- 'boolField': True,
- 'floatField': 1234.0,
- 'doubleField': -1234.0,
- 'bytesField': b'12312adf',
- 'nullField': None,
- 'arrayField': [5.0, 0.0, 12.0],
- 'mapField': {'a': {'label': 'a'}, 'bee': {'label': 'cee'}},
- 'unionField': 12.0,
- 'enumField': 'C',
- 'fixedField': b'1019181716151413',
- 'recordField': {
- 'label': 'blah',
- 'children': [{'label': 'inner', 'children': []}],
- },
-}
-
-
-def WriteDataFile(path, datum, schema):
- datum_writer = io.DatumWriter()
- with open(path, 'wb') as writer:
- # NB: not using compression
- with datafile.DataFileWriter(writer, datum_writer, schema) as dfw:
- dfw.append(datum)
class TestDataFileInterop(unittest.TestCase):
def testInterop(self):
- with tempfile.NamedTemporaryFile() as temp_path:
- WriteDataFile(temp_path.name, INTEROP_DATUM, INTEROP_SCHEMA)
-
- # read data in binary from file
- datum_reader = io.DatumReader()
- with open(temp_path.name, 'rb') as reader:
- dfr = datafile.DataFileReader(reader, datum_reader)
- for datum in dfr:
- self.assertEqual(INTEROP_DATUM, datum)
+ datum_reader = io.DatumReader()
+ for avro_file in glob.glob('../../build/interop/data/*.avro'):
+ base_ext = os.path.splitext(os.path.basename(avro_file))[0].split('_', 1)
+ if len(base_ext) < 2 or base_ext[1] in datafile.VALID_CODECS:
+ with open(avro_file, 'rb') as reader, \
+ datafile.DataFileReader(reader, datum_reader) as dfr:
+ i = 0
+ for i, datum in enumerate(dfr, 1):
+ self.assertIsNotNone(datum)
+ self.assertGreater(i, 0)
if __name__ == '__main__':
diff --git a/lang/py3/setup.py b/lang/py3/setup.py
index a752779..22b6472 100755
--- a/lang/py3/setup.py
+++ b/lang/py3/setup.py
@@ -26,9 +26,10 @@ The avro-python3 software is designed for Python 3, but this file and the packag
https://pypi.org/project/avro-python3/
"""
+import distutils.cmd
import distutils.command.clean
-import distutils.file_util
import distutils.dir_util
+import distutils.file_util
import distutils.log
import fnmatch
import os
@@ -110,11 +111,34 @@ class CleanCommand(distutils.command.clean.clean):
os.remove(name)
+class GenerateInteropDataCommand(distutils.cmd.Command):
+ """A command to generate Avro files for data interop test."""
+
+ user_options = [
+ ('schema-file=', None, 'path to input Avro schema file'),
+ ('output-path=', None, 'path to output Avro data files'),
+ ]
+
+ def initialize_options(self):
+ self.schema_file = os.path.join(os.getcwd(), 'interop.avsc')
+ self.output_path = os.getcwd()
+
+ def finalize_options(self):
+ pass
+
+ def run(self):
+ from avro.tests import gen_interop_data
+ gen_interop_data.generate(self.schema_file, self.output_path)
+
+
def main():
if not _is_distribution():
_generate_package_data()
- setup(cmdclass={"clean": CleanCommand})
+ setup(cmdclass={
+ "clean": CleanCommand,
+ "generate_interop_data": GenerateInteropDataCommand,
+ })
if __name__ == '__main__':