You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by jm...@apache.org on 2010/06/04 07:28:11 UTC
svn commit: r951270 - in /avro/branches/branch-1.3: ./
lang/ruby/lib/avro/data_file.rb lang/ruby/lib/avro/io.rb
lang/ruby/test/test_datafile.rb
Author: jmhodges
Date: Fri Jun 4 05:28:11 2010
New Revision: 951270
URL: http://svn.apache.org/viewvc?rev=951270&view=rev
Log:
AVRO-489. Skipping complex objects in the ruby impl. merge from trunk
Modified:
avro/branches/branch-1.3/ (props changed)
avro/branches/branch-1.3/lang/ruby/lib/avro/data_file.rb
avro/branches/branch-1.3/lang/ruby/lib/avro/io.rb
avro/branches/branch-1.3/lang/ruby/test/test_datafile.rb
Propchange: avro/branches/branch-1.3/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Jun 4 05:28:11 2010
@@ -1,2 +1,2 @@
/avro/trunk:944035,944049
-/hadoop/avro/trunk:930458-930459,935526,938347
+/hadoop/avro/trunk:930458-930459,930461,935526,938347
Modified: avro/branches/branch-1.3/lang/ruby/lib/avro/data_file.rb
URL: http://svn.apache.org/viewvc/avro/branches/branch-1.3/lang/ruby/lib/avro/data_file.rb?rev=951270&r1=951269&r2=951270&view=diff
==============================================================================
--- avro/branches/branch-1.3/lang/ruby/lib/avro/data_file.rb (original)
+++ avro/branches/branch-1.3/lang/ruby/lib/avro/data_file.rb Fri Jun 4 05:28:11 2010
@@ -46,7 +46,7 @@ module Avro
yield io if block_given?
io
ensure
- io.close if block_given?
+ io.close if block_given? && io
end
class << self
Modified: avro/branches/branch-1.3/lang/ruby/lib/avro/io.rb
URL: http://svn.apache.org/viewvc/avro/branches/branch-1.3/lang/ruby/lib/avro/io.rb?rev=951270&r1=951269&r2=951270&view=diff
==============================================================================
--- avro/branches/branch-1.3/lang/ruby/lib/avro/io.rb (original)
+++ avro/branches/branch-1.3/lang/ruby/lib/avro/io.rb Fri Jun 4 05:28:11 2010
@@ -516,6 +516,42 @@ module Avro
raise AvroError, "Unknown schema type: #{schm.type}"
end
end
+
+ def skip_fixed(writers_schema, decoder)
+ decoder.skip(writers_schema.size)
+ end
+
+ def skip_enum(writers_schema, decoder)
+ decoder.skip_int
+ end
+
+ def skip_array(writers_schema, decoder)
+ skip_blocks(decoder) { skip_data(writers_schema.items, decoder) }
+ end
+
+ def skip_map(writers_schema, decoder)
+ skip_blocks(decoder) {
+ decoder.skip_string
+ skip_data(writers_schema.values, decoder)
+ }
+ end
+
+ def skip_record(writers_schema, decoder)
+ writers_schema.fields.each{|f| skip_data(f.type, decoder) }
+ end
+
+ private
+ def skip_blocks(decoder, &blk)
+ block_count = decoder.read_long
+ while block_count != 0
+ if block_count < 0
+ decoder.skip(decoder.read_long)
+ else
+ block_count.times &blk
+ end
+ block_count = decoder.read_long
+ end
+ end
end # DatumReader
# DatumWriter for generic ruby objects
Modified: avro/branches/branch-1.3/lang/ruby/test/test_datafile.rb
URL: http://svn.apache.org/viewvc/avro/branches/branch-1.3/lang/ruby/test/test_datafile.rb?rev=951270&r1=951269&r2=951270&view=diff
==============================================================================
--- avro/branches/branch-1.3/lang/ruby/test/test_datafile.rb (original)
+++ avro/branches/branch-1.3/lang/ruby/test/test_datafile.rb Fri Jun 4 05:28:11 2010
@@ -48,4 +48,58 @@ JSON
end
end
+ def test_differing_schemas_with_complex_objects
+ writer_schema = <<-JSON
+{ "type": "record",
+ "name": "something",
+ "fields": [
+ {"name": "something_fixed", "type": {"name": "inner_fixed",
+ "type": "fixed", "size": 3}},
+ {"name": "something_enum", "type": {"name": "inner_enum",
+ "type": "enum",
+ "symbols": ["hello", "goodbye"]}},
+ {"name": "something_array", "type": {"type": "array", "items": "int"}},
+ {"name": "something_map", "type": {"type": "map", "values": "int"}},
+ {"name": "something_record", "type": {"name": "inner_record",
+ "type": "record",
+ "fields": [
+ {"name": "inner", "type": "int"}
+ ]}},
+ {"name": "username", "type": "string"}
+]}
+JSON
+
+ data = [{"username" => "john",
+ "something_fixed" => "foo",
+ "something_enum" => "hello",
+ "something_array" => [1,2,3],
+ "something_map" => {"a" => 1, "b" => 2},
+ "something_record" => {"inner" => 2},
+ "something_error" => {"code" => 403}
+ },
+ {"username" => "ryan",
+ "something_fixed" => "bar",
+ "something_enum" => "goodbye",
+ "something_array" => [1,2,3],
+ "something_map" => {"a" => 2, "b" => 6},
+ "something_record" => {"inner" => 1},
+ "something_error" => {"code" => 401}
+ }]
+
+ Avro::DataFile.open('data.avr', 'w', writer_schema) do |dw|
+ data.each{|d| dw << d }
+ end
+
+ %w[fixed enum record error array map union].each do |s|
+ reader = Yajl.load(writer_schema)
+ reader['fields'] = reader['fields'].reject{|f| f['type']['type'] == s}
+ Avro::DataFile.open('data.avr', 'r', Yajl.dump(reader)) do |dr|
+ dr.each_with_index do |obj, i|
+ reader['fields'].each do |field|
+ assert_equal data[i][field['name']], obj[field['name']]
+ end
+ end
+ end
+ end
+ end
end