You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by ph...@apache.org on 2010/08/20 17:22:12 UTC
svn commit: r987543 - in /avro/trunk: CHANGES.txt lang/py/build.xml
lang/py/src/avro/datafile.py lang/py/src/avro/io.py
lang/py/src/avro/protocol.py lang/py/test/test_protocol.py
lang/py/test/test_schema.py
Author: philz
Date: Fri Aug 20 15:22:10 2010
New Revision: 987543
URL: http://svn.apache.org/viewvc?rev=987543&view=rev
Log:
AVRO-618. Avro doesn't work with python 2.4
Modified:
avro/trunk/CHANGES.txt
avro/trunk/lang/py/build.xml
avro/trunk/lang/py/src/avro/datafile.py
avro/trunk/lang/py/src/avro/io.py
avro/trunk/lang/py/src/avro/protocol.py
avro/trunk/lang/py/test/test_protocol.py
avro/trunk/lang/py/test/test_schema.py
Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=987543&r1=987542&r2=987543&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Fri Aug 20 15:22:10 2010
@@ -129,6 +129,8 @@ Avro 1.4.0 (unreleased)
BUG FIXES
+ AVRO-618. Avro doesn't work with python 2.4 (philz)
+
AVRO-502. Memory leak from parsing JSON schema.
(Robert G. Jakabosky via massie)
Modified: avro/trunk/lang/py/build.xml
URL: http://svn.apache.org/viewvc/avro/trunk/lang/py/build.xml?rev=987543&r1=987542&r2=987543&view=diff
==============================================================================
--- avro/trunk/lang/py/build.xml (original)
+++ avro/trunk/lang/py/build.xml Fri Aug 20 15:22:10 2010
@@ -28,6 +28,8 @@
<property name="top.build" value="${basedir}/../../build"/>
<property name="interop.data.dir" value="${top.build}/interop/data"/>
+ <property name="python" value="python"/>
+
<!-- Python implementation directories -->
<property name="build.dir" value="${basedir}/build"/>
<property name="src.dir" value="${basedir}/src"/>
@@ -111,7 +113,7 @@
depends="build">
<taskdef name="py-test" classname="org.pyant.tasks.PythonTestTask"
classpathref="java.classpath"/>
- <py-test python="python" pythonpathref="test.path" >
+ <py-test python="${python}" pythonpathref="test.path" >
<fileset dir="${build.dir}/test">
<include name="test_*.py"/>
<exclude name="test_datafile_interop.py"/>
@@ -125,7 +127,7 @@
depends="build">
<taskdef name="py-test" classname="org.pyant.tasks.PythonTestTask"
classpathref="java.classpath"/>
- <py-test python="python" pythonpathref="test.path" >
+ <py-test python="${python}" pythonpathref="test.path" >
<fileset dir="${build.dir}/test">
<include name="test_datafile_interop.py"/>
</fileset>
@@ -137,7 +139,7 @@
description="Generate Python interop data files."
depends="build">
<mkdir dir="${interop.data.dir}"/>
- <exec executable="python">
+ <exec executable="${python}">
<env key="PYTHONPATH" value="$PYTHONPATH:${build.dir}/src"/>
<arg value="${build.dir}/test/gen_interop_data.py"/>
<arg value="${share.dir}/test/schemas/interop.avsc"/>
@@ -150,7 +152,7 @@
description="Build source distribution"
depends="build">
<mkdir dir="${dist.dir}"/>
- <exec executable="python" failonerror="true">
+ <exec executable="${python}" failonerror="true">
<arg value="${build.dir}/setup.py"/>
<arg value="sdist"/>
<arg value="--dist-dir=${dist.dir}"/>
Modified: avro/trunk/lang/py/src/avro/datafile.py
URL: http://svn.apache.org/viewvc/avro/trunk/lang/py/src/avro/datafile.py?rev=987543&r1=987542&r2=987543&view=diff
==============================================================================
--- avro/trunk/lang/py/src/avro/datafile.py (original)
+++ avro/trunk/lang/py/src/avro/datafile.py Fri Aug 20 15:22:10 2010
@@ -17,7 +17,6 @@
Read/Write Avro File Object Containers.
"""
import zlib
-import uuid
try:
from cStringIO import StringIO
except ImportError:
@@ -65,7 +64,7 @@ class DataFileException(schema.AvroExcep
class DataFileWriter(object):
@staticmethod
def generate_sync_marker():
- return uuid.uuid4().bytes
+ return generate_sixteen_random_bytes()
# TODO(hammer): make 'encoder' a metadata property
def __init__(self, writer, datum_writer, writers_schema=None, codec='null'):
@@ -322,3 +321,11 @@ class DataFileReader(object):
def close(self):
"""Close this reader."""
self.reader.close()
+
+def generate_sixteen_random_bytes():
+ try:
+ import os
+ return os.urandom(16)
+ except:
+ import random
+ return [ chr(random.randrange(256)) for i in range(16) ]
Modified: avro/trunk/lang/py/src/avro/io.py
URL: http://svn.apache.org/viewvc/avro/trunk/lang/py/src/avro/io.py?rev=987543&r1=987542&r2=987543&view=diff
==============================================================================
--- avro/trunk/lang/py/src/avro/io.py (original)
+++ avro/trunk/lang/py/src/avro/io.py Fri Aug 20 15:22:10 2010
@@ -38,6 +38,7 @@ uses the following mapping:
"""
import struct
from avro import schema
+import sys
#
# Constants
@@ -49,10 +50,22 @@ LONG_MIN_VALUE = -(1 << 63)
LONG_MAX_VALUE = (1 << 63) - 1
# TODO(hammer): shouldn't ! be < for little-endian (according to spec?)
-STRUCT_INT = struct.Struct('!I') # big-endian unsigned int
-STRUCT_LONG = struct.Struct('!Q') # big-endian unsigned long long
-STRUCT_FLOAT = struct.Struct('!f') # big-endian float
-STRUCT_DOUBLE = struct.Struct('!d') # big-endian double
+if sys.version_info >= (2, 5, 0):
+ struct_class = struct.Struct
+else:
+ class SimpleStruct(object):
+ def __init__(self, format):
+ self.format = format
+ def pack(self, *args):
+ return struct.pack(self.format, *args)
+ def unpack(self, *args):
+ return struct.unpack(self.format, *args)
+ struct_class = SimpleStruct
+
+STRUCT_INT = struct_class('!I') # big-endian unsigned int
+STRUCT_LONG = struct_class('!Q') # big-endian unsigned long long
+STRUCT_FLOAT = struct_class('!f') # big-endian float
+STRUCT_DOUBLE = struct_class('!d') # big-endian double
#
# Exceptions
Modified: avro/trunk/lang/py/src/avro/protocol.py
URL: http://svn.apache.org/viewvc/avro/trunk/lang/py/src/avro/protocol.py?rev=987543&r1=987542&r2=987543&view=diff
==============================================================================
--- avro/trunk/lang/py/src/avro/protocol.py (original)
+++ avro/trunk/lang/py/src/avro/protocol.py Fri Aug 20 15:22:10 2010
@@ -17,9 +17,9 @@
Protocol implementation.
"""
try:
- import hashlib
+ from hashlib import md5
except ImportError:
- import md5
+ from md5 import md5
try:
import json
except ImportError:
@@ -100,10 +100,7 @@ class Protocol(object):
self.set_prop('types', self._parse_types(types, type_names))
if messages is not None:
self.set_prop('messages', self._parse_messages(messages, type_names))
- if hashlib:
- self._md5 = hashlib.md5(str(self)).digest()
- else:
- self._md5 = md5.new(str(self)).digest()
+ self._md5 = md5(str(self)).digest()
# read-only properties
name = property(lambda self: self.get_prop('name'))
Modified: avro/trunk/lang/py/test/test_protocol.py
URL: http://svn.apache.org/viewvc/avro/trunk/lang/py/test/test_protocol.py?rev=987543&r1=987542&r2=987543&view=diff
==============================================================================
--- avro/trunk/lang/py/test/test_protocol.py (original)
+++ avro/trunk/lang/py/test/test_protocol.py Fri Aug 20 15:22:10 2010
@@ -318,12 +318,13 @@ class TestProtocol(unittest.TestCase):
num_correct = 0
for example in EXAMPLES:
try:
- protocol.parse(example.protocol_string)
- if example.valid: num_correct += 1
- debug_msg = "%s: PARSE SUCCESS" % example.name
- except:
- if not example.valid: num_correct += 1
- debug_msg = "%s: PARSE FAILURE" % example.name
+ try:
+ protocol.parse(example.protocol_string)
+ if example.valid: num_correct += 1
+ debug_msg = "%s: PARSE SUCCESS" % example.name
+ except:
+ if not example.valid: num_correct += 1
+ debug_msg = "%s: PARSE FAILURE" % example.name
finally:
print debug_msg
@@ -345,11 +346,12 @@ class TestProtocol(unittest.TestCase):
for example in VALID_EXAMPLES:
protocol_data = protocol.parse(example.protocol_string)
try:
- protocol.parse(str(protocol_data))
- debug_msg = "%s: STRING CAST SUCCESS" % example.name
- num_correct += 1
- except:
- debug_msg = "%s: STRING CAST FAILURE" % example.name
+ try:
+ protocol.parse(str(protocol_data))
+ debug_msg = "%s: STRING CAST SUCCESS" % example.name
+ num_correct += 1
+ except:
+ debug_msg = "%s: STRING CAST FAILURE" % example.name
finally:
print debug_msg
@@ -372,16 +374,17 @@ class TestProtocol(unittest.TestCase):
num_correct = 0
for example in VALID_EXAMPLES:
try:
- original_protocol = protocol.parse(example.protocol_string)
- round_trip_protocol = protocol.parse(str(original_protocol))
-
- if original_protocol == round_trip_protocol:
- num_correct += 1
- debug_msg = "%s: ROUND TRIP SUCCESS" % example.name
- else:
+ try:
+ original_protocol = protocol.parse(example.protocol_string)
+ round_trip_protocol = protocol.parse(str(original_protocol))
+
+ if original_protocol == round_trip_protocol:
+ num_correct += 1
+ debug_msg = "%s: ROUND TRIP SUCCESS" % example.name
+ else:
+ debug_msg = "%s: ROUND TRIP FAILURE" % example.name
+ except:
debug_msg = "%s: ROUND TRIP FAILURE" % example.name
- except:
- debug_msg = "%s: ROUND TRIP FAILURE" % example.name
finally:
print debug_msg
Modified: avro/trunk/lang/py/test/test_schema.py
URL: http://svn.apache.org/viewvc/avro/trunk/lang/py/test/test_schema.py?rev=987543&r1=987542&r2=987543&view=diff
==============================================================================
--- avro/trunk/lang/py/test/test_schema.py (original)
+++ avro/trunk/lang/py/test/test_schema.py Fri Aug 20 15:22:10 2010
@@ -275,12 +275,19 @@ class TestSchema(unittest.TestCase):
correct = 0
for example in EXAMPLES:
try:
- schema.parse(example.schema_string)
- if example.valid: correct += 1
- debug_msg = "%s: PARSE SUCCESS" % example.name
- except:
- if not example.valid: correct += 1
- debug_msg = "%s: PARSE FAILURE" % example.name
+ try:
+ schema.parse(example.schema_string)
+ if example.valid:
+ correct += 1
+ else:
+ self.fail("Invalid schema was parsed: " + example.schema_string)
+ debug_msg = "%s: PARSE SUCCESS" % example.name
+ except:
+ if not example.valid:
+ correct += 1
+ else:
+ self.fail("Valid schema failed to parse: " + example.schema_string)
+ debug_msg = "%s: PARSE FAILURE" % example.name
finally:
print debug_msg
@@ -298,11 +305,12 @@ class TestSchema(unittest.TestCase):
for example in VALID_EXAMPLES:
schema_data = schema.parse(example.schema_string)
try:
- schema.parse(str(schema_data))
- debug_msg = "%s: STRING CAST SUCCESS" % example.name
- correct += 1
- except:
- debug_msg = "%s: STRING CAST FAILURE" % example.name
+ try:
+ schema.parse(str(schema_data))
+ debug_msg = "%s: STRING CAST SUCCESS" % example.name
+ correct += 1
+ except:
+ debug_msg = "%s: STRING CAST FAILURE" % example.name
finally:
print debug_msg
@@ -321,15 +329,16 @@ class TestSchema(unittest.TestCase):
correct = 0
for example in VALID_EXAMPLES:
try:
- original_schema = schema.parse(example.schema_string)
- round_trip_schema = schema.parse(str(original_schema))
- if original_schema == round_trip_schema:
- correct += 1
- debug_msg = "%s: ROUND TRIP SUCCESS" % example.name
- else:
+ try:
+ original_schema = schema.parse(example.schema_string)
+ round_trip_schema = schema.parse(str(original_schema))
+ if original_schema == round_trip_schema:
+ correct += 1
+ debug_msg = "%s: ROUND TRIP SUCCESS" % example.name
+ else:
+ debug_msg = "%s: ROUND TRIP FAILURE" % example.name
+ except:
debug_msg = "%s: ROUND TRIP FAILURE" % example.name
- except:
- debug_msg = "%s: ROUND TRIP FAILURE" % example.name
finally:
print debug_msg