You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by dc...@apache.org on 2012/03/01 05:01:09 UTC
svn commit: r1295404 - in /avro/trunk: CHANGES.txt
lang/c/src/resolved-reader.c lang/c/tests/CMakeLists.txt
lang/c/tests/test_avro_1034.c
Author: dcreager
Date: Thu Mar 1 04:01:09 2012
New Revision: 1295404
URL: http://svn.apache.org/viewvc?rev=1295404&view=rev
Log:
AVRO-1034. C: Resolved reader initializes child array.
The resolved reader array implementation now initializes its children
array. We only need to initialize the elements of the array when
they're first created. This happens the first time we access a
particular element, or any element that appears after it in the array.
Test case contributed by Vivek Nadkarni.
Added:
avro/trunk/lang/c/tests/test_avro_1034.c
Modified:
avro/trunk/CHANGES.txt
avro/trunk/lang/c/src/resolved-reader.c
avro/trunk/lang/c/tests/CMakeLists.txt
Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1295404&r1=1295403&r2=1295404&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Thu Mar 1 04:01:09 2012
@@ -21,6 +21,9 @@ Avro 1.6.3 (unreleased)
AVRO-1033. C: Fixed x86 assembly implementation of atomic reference
counting primitives. (Vivek Nadkarni via dcreager)
+ AVRO-1034. C: Resolved readers initialize complex array values
+ correctly. (Vivek Nadkarni via dcreager)
+
AVRO-1038. C: Require GCC 4.5.0 to use GCC atomic instrinsics.
(Vivek Nadkarni via dcreager)
@@ -129,7 +132,7 @@ Avro 1.6.2 (13 February 2012)
AVRO-982. Java: Fix NettyTransceiver to not hang when server stops.
(Bruno Dumon via cutting)
- AVRO-984. C: Resolved readers initialize complex array values
+ AVRO-984. C: Resolved writers initialize complex array values
correctly. (Vivek Nadkarni via dcreager)
AVRO-994. Java: TestFileSpanStorage.testTonsOfSpans() fails on my
Modified: avro/trunk/lang/c/src/resolved-reader.c
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c/src/resolved-reader.c?rev=1295404&r1=1295403&r2=1295404&view=diff
==============================================================================
--- avro/trunk/lang/c/src/resolved-reader.c (original)
+++ avro/trunk/lang/c/src/resolved-reader.c Thu Mar 1 04:01:09 2012
@@ -1500,6 +1500,7 @@ avro_resolved_array_reader_reset(const a
/* Clear out our cache of wrapped children */
avro_resolved_array_reader_free_elements(aiface->child_resolver, self);
+ avro_raw_array_clear(&self->children);
return 0;
}
@@ -1518,6 +1519,8 @@ avro_resolved_array_reader_get_by_index(
avro_value_t *child, const char **name)
{
int rval;
+ size_t old_size;
+ size_t new_size;
const avro_resolved_reader_t *iface =
container_of(viface, avro_resolved_reader_t, parent);
const avro_resolved_array_reader_t *aiface =
@@ -1528,8 +1531,16 @@ avro_resolved_array_reader_get_by_index(
* Ensure that our child wrapper array is big enough to hold
* this many elements.
*/
- check(rval, avro_raw_array_ensure_size0(&self->children, index+1));
- if (avro_raw_array_size(&self->children) <= index) {
+ new_size = index + 1;
+ check(rval, avro_raw_array_ensure_size0(&self->children, new_size));
+ old_size = avro_raw_array_size(&self->children);
+ if (old_size <= index) {
+ size_t i;
+ for (i = old_size; i < new_size; i++) {
+ check(rval, avro_resolved_reader_init
+ (aiface->child_resolver,
+ avro_raw_array_get_raw(&self->children, i)));
+ }
avro_raw_array_size(&self->children) = index+1;
}
Modified: avro/trunk/lang/c/tests/CMakeLists.txt
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c/tests/CMakeLists.txt?rev=1295404&r1=1295403&r2=1295404&view=diff
==============================================================================
--- avro/trunk/lang/c/tests/CMakeLists.txt (original)
+++ avro/trunk/lang/c/tests/CMakeLists.txt Thu Mar 1 04:01:09 2012
@@ -44,6 +44,7 @@ add_avro_test(test_avro_schema_names)
add_avro_test(test_avro_values)
add_avro_test(test_avro_968)
add_avro_test(test_avro_984)
+add_avro_test(test_avro_1034)
add_avro_test(test_avro_data)
add_avro_test(test_refcount)
add_avro_test(test_cpp test_cpp.cpp)
Added: avro/trunk/lang/c/tests/test_avro_1034.c
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c/tests/test_avro_1034.c?rev=1295404&view=auto
==============================================================================
--- avro/trunk/lang/c/tests/test_avro_1034.c (added)
+++ avro/trunk/lang/c/tests/test_avro_1034.c Thu Mar 1 04:01:09 2012
@@ -0,0 +1,395 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+#include <avro.h>
+#include <inttypes.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+/* Test code for JIRA Issue AVRO-1034.
+ *
+ * AVRO-1034: Resolved reader does not initialize children of arrays,
+ * resulting in seg faults
+ *
+ * This program tests schema resolution for nested arrays. For the
+ * purposes of this test, there are two schemas "old" and "new" which
+ * are created by reading the same JSON schema.
+ *
+ * The test creates and populates a nested array avro value, and
+ * serializes it to memory. The raw memory is written to a file. Note
+ * that the schema is not written to the file. The nested array is
+ * also printed to the screen.
+ *
+ * An identical nested array avro value is then created. A
+ * resolved_reader_class and a corresponding resolved_record instance
+ * is created (using identical "writer" and "reader" schemas for
+ * simplicity), and an attempt is made to "read" the resolved avro
+ * value.
+ *
+ * Once the resolved value has been read, the source value (nested)
+ * and the resolved value (resolved_record) are both reset using
+ * avro_value_reset(). Then the source value (nested) is populated
+ * with another (larger) nested array. Then an attempt is made to read
+ * the resolved avro value again.
+ *
+ * This second attempt to read the resolved value results in a
+ * segmentation fault under Linux, using the patch in
+ * https://issues.apache.org/jira/secure/attachment/12516487/0001-AVRO-1034.-C-Resolved-reader-initializes-child-array.patch.
+ *
+ * However, the program does not seg fault, using the patch in
+ * https://issues.apache.org/jira/secure/attachment/12515544/AVRO-1034.patch
+ *
+ * AVRO-C was compiled with CMAKE_INSTALL_PREFIX=avrolib
+ * The static library (libavro.a) was copied into a subdirectory of avrolib/lib/static
+ *
+ * This file was compiled under Linux using:
+ * gcc -g avro-1034-test-2.c -o test2 -I../../build/avrolib/include -L../../build/avrolib/lib/static -lavro
+ *
+ */
+
+
+// Encode the following json string in NESTED_ARRAY
+// {"type":"array", "items": {"type": "array", "items": "long"}}
+//
+#define NESTED_ARRAY \
+ "{\"type\":\"array\", \"items\": {\"type\": \"array\", \"items\": \"long\"}}"
+
+avro_schema_t schema_old = NULL;
+avro_schema_t schema_new = NULL;
+
+/* Parse schema into a schema data structure */
+void init_schema(void)
+{
+ avro_schema_error_t error;
+ if (avro_schema_from_json(NESTED_ARRAY, sizeof(NESTED_ARRAY),
+ &schema_old, &error)) {
+ printf( "Unable to parse old schema\n");
+ exit(EXIT_FAILURE);
+ }
+
+ if (avro_schema_from_json(NESTED_ARRAY, sizeof(NESTED_ARRAY),
+ &schema_new, &error)) {
+ printf( "Unable to parse new schema\n");
+ exit(EXIT_FAILURE);
+ }
+}
+
+#define try(call, msg) \
+ do { \
+ if (call) { \
+ printf( msg ":\n %s\n", avro_strerror()); \
+ exit (EXIT_FAILURE); \
+ } \
+ } while (0)
+
+
+/* The input avro_value_t p_array should contain a nested array.
+ * Print the fields of this nested array to the screen.
+ */
+int print_array_fields ( avro_value_t *p_array )
+{
+ size_t idx;
+ size_t length;
+ avro_type_t val_type;
+
+ val_type = avro_value_get_type( p_array );
+ printf( "Main array type = %d\n", val_type );
+
+ try( avro_value_get_size( p_array, &length ),
+ "Couldn't get array size" );
+ printf( "Main array length = %d\n", (int) length );
+
+ for ( idx = 0; idx < length; idx ++ )
+ {
+ avro_value_t subarray;
+ size_t sublength;
+ size_t jdx;
+ const char *unused;
+
+ try ( avro_value_get_by_index( p_array, idx, &subarray, &unused ),
+ "Couldn't get subarray" );
+
+ val_type = avro_value_get_type( &subarray );
+ printf( "Subarray type = %d\n", val_type );
+
+ try( avro_value_get_size( &subarray, &sublength ),
+ "Couldn't get subarray size" );
+ printf( "Subarray length = %d\n", (int) sublength );
+
+ for ( jdx = 0; jdx < sublength; jdx++ )
+ {
+ avro_value_t element;
+ int64_t val;
+
+ try ( avro_value_get_by_index( &subarray, jdx, &element, &unused ),
+ "Couldn't get subarray element" );
+
+ val_type = avro_value_get_type( &element );
+
+ try ( avro_value_get_long( &element, &val ),
+ "Couldn't get subarray element value" );
+
+ printf( "nested_array[%d][%d]: type = %d value = %lld\n",
+ (int) idx, (int) jdx, (int) val_type, (long long) val );
+
+ }
+ }
+
+ return 0;
+}
+
+
+/* The input avro_value_t p_subarray should contain an array of long
+ * integers. Add "elements" number of long integers to this array. Set
+ * the values to be distinct based on the iteration parameter.
+ */
+int add_subarray( avro_value_t *p_subarray,
+ size_t elements,
+ int32_t iteration )
+{
+ avro_value_t element;
+ size_t index;
+ size_t idx;
+
+ for ( idx = 0; idx < elements; idx ++ )
+ {
+ // Append avro array element to subarray
+ try ( avro_value_append( p_subarray, &element, &index ),
+ "Error appending element in subarray" );
+
+ try ( avro_value_set_long( &element, (iteration+1)*100 + (iteration+1) ),
+ "Error setting subarray element" );
+ }
+
+ return 0;
+}
+
+int populate_array( avro_value_t *p_array, int32_t elements )
+{
+ int32_t idx;
+ fprintf( stderr, "Elements = %d\n", elements);
+ for ( idx = 0; idx < elements; idx ++ )
+ {
+ avro_value_t subarray;
+ size_t index;
+
+ // Append avro array element for top level array
+ try ( avro_value_append( p_array, &subarray, &index ),
+ "Error appending subarray" );
+
+ // Populate array element with subarray of length 2
+#define SUBARRAY_LENGTH (2)
+ try ( add_subarray( &subarray, SUBARRAY_LENGTH, idx ),
+ "Error populating subarray" );
+ }
+ return 0;
+}
+
+
+/* Create a nested array using the schema NESTED_ARRAY. Populate its
+ * elements with unique values. Serialize the nested array to the
+ * memory buffer in avro_writer_t. The number of elements in the first
+ * dimension of the nested array is "elements". The number of elements
+ * in the second dimension of the nested array is hardcoded to 2.
+ */
+int add_array( avro_writer_t writer,
+ int32_t elements,
+ int use_resolving_writer )
+{
+ avro_schema_t chosen_schema;
+ avro_value_iface_t *nested_array_class;
+ avro_value_t nested;
+
+ // Select (hardcode) schema to use
+ chosen_schema = schema_old;
+
+ // Create avro class and value
+ nested_array_class = avro_generic_class_from_schema( chosen_schema );
+ try ( avro_generic_value_new( nested_array_class, &nested ),
+ "Error creating instance of record" );
+
+ try ( populate_array( &nested, elements ),
+ "Error populating array" );
+
+ if ( use_resolving_writer )
+ {
+ // Resolve schema differences
+ avro_value_iface_t *resolved_reader_class;
+ avro_value_iface_t *writer_class;
+ avro_value_t resolved_record;
+
+ // Note - we will read values from the reader of "schema to write
+ // to file" and we will copy them into a writer of the same
+ // schema.
+ resolved_reader_class = avro_resolved_reader_new( schema_old,// schema populated above
+ schema_new // schema_to_write_to_file
+ );
+ if ( resolved_reader_class == NULL )
+ {
+ printf( "Failed avro_resolved_reader_new()\n");
+ exit( EXIT_FAILURE );
+ }
+
+ try ( avro_resolved_reader_new_value( resolved_reader_class, &resolved_record ),
+ "Failed avro_resolved_reader_new_value" );
+
+ // Map the resolved reader to the record you want to get data from
+ avro_resolved_reader_set_source( &resolved_record, &nested );
+
+ // Now the resolved_record is mapped to read data from record. Now
+ // we need to copy the data from resolved_record into a
+ // writer_record, which is an instance of the same schema as
+ // resolved_record.
+
+ // Create a writer of the schema you want to write using
+ writer_class = avro_generic_class_from_schema( schema_new );
+ if ( writer_class == NULL )
+ {
+ printf( "Failed avro_generic_class_from_schema()\n");
+ exit( EXIT_FAILURE );
+ }
+
+ try ( avro_value_write( writer, &resolved_record ),
+ "Unable to write record into memory using writer_record" );
+
+ print_array_fields( &resolved_record );
+
+ avro_value_reset( &nested );
+
+ // Question: Is it permissible to call avro_value_reset() on a
+ // resolved_record? Set the #if 1 to #if 0 to disable the
+ // avro_value_reset(), to prevent the segmentation fault.
+ #if 1
+ avro_value_reset( &resolved_record );
+ #endif
+
+ try ( populate_array( &nested, 2*elements ),
+ "Error populating array" );
+
+ try ( avro_value_write( writer, &resolved_record ),
+ "Unable to write record into memory using writer_record" );
+
+ print_array_fields( &resolved_record );
+
+ avro_value_decref( &resolved_record );
+ avro_value_iface_decref( writer_class );
+ }
+ else
+ {
+ // Write the value to memory
+ try ( avro_value_write( writer, &nested ),
+ "Unable to write nested into memory" );
+
+ print_array_fields( &nested );
+ }
+
+
+ // Release the record
+ avro_value_decref( &nested );
+ avro_value_iface_decref( nested_array_class );
+
+ return 0;
+}
+
+/* Create a raw binary file containing a serialized version of a
+ * nested array. This file will later be read by
+ * read_nested_array_file().
+ */
+int write_nested_array_file ( int64_t buf_len,
+ const char *raw_binary_file_name,
+ int use_resolving_writer )
+{
+ char *buf;
+ avro_writer_t nested_writer;
+ FILE *fid = NULL;
+
+ fprintf( stdout, "Create %s\n", raw_binary_file_name );
+
+ // Allocate a buffer
+ buf = malloc( buf_len * sizeof( char ) );
+ if ( buf == NULL )
+ {
+ printf( "There was an error creating the nested buffer %s.\n", raw_binary_file_name);
+ exit(EXIT_FAILURE);
+ }
+
+ /* Create a new memory writer */
+ nested_writer = avro_writer_memory( buf, buf_len );
+ if ( nested_writer == NULL )
+ {
+ printf( "There was an error creating the buffer for writing %s.\n", raw_binary_file_name);
+ exit(EXIT_FAILURE);
+ }
+
+ /* Add an array containing 4 subarrays */
+ printf( "before avro_writer_tell %d\n", (int) avro_writer_tell( nested_writer ) );
+#define ARRAY_LENGTH (4)
+ add_array( nested_writer, ARRAY_LENGTH, use_resolving_writer );
+ printf( "after avro_writer_tell %d\n", (int) avro_writer_tell( nested_writer ) );
+
+ /* Serialize the nested array */
+ printf( "Serialize the data to a file\n");
+
+ /* Delete the nested array if it exists, and create a new one */
+ unlink(raw_binary_file_name);
+ fid = fopen( raw_binary_file_name, "w+");
+ if ( fid == NULL )
+ {
+ printf( "There was an error creating the file %s.\n", raw_binary_file_name);
+ exit(EXIT_FAILURE);
+ }
+ fwrite( buf, 1, avro_writer_tell( nested_writer ), fid );
+ fclose(fid);
+ avro_writer_free( nested_writer );
+ free(buf);
+ return 0;
+}
+
+
+/* Top level function to impelement a test for the JIRA issue
+ * AVRO-1034. See detailed documentation at the top of this file.
+ */
+int main(void)
+{
+ const char *raw_binary_file_name = "nested_array.bin";
+ const char *raw_binary_file_name_resolved = "nested_array_resolved.bin";
+ int64_t buf_len = 2048;
+ int use_resolving_writer;
+
+ /* Initialize the schema structure from JSON */
+ init_schema();
+
+ printf( "Write the serialized nested array to %s\n", raw_binary_file_name );
+ use_resolving_writer = 0;
+ write_nested_array_file( buf_len, raw_binary_file_name, use_resolving_writer );
+
+ printf( "\nWrite the serialized nested array after schema resolution to %s\n",
+ raw_binary_file_name_resolved );
+ use_resolving_writer = 1;
+ write_nested_array_file( buf_len, raw_binary_file_name_resolved, use_resolving_writer );
+
+ // Close out schemas
+ avro_schema_decref(schema_old);
+ avro_schema_decref(schema_new);
+
+ // Remove the binary file
+ // unlink(raw_binary_file_name);
+
+ printf("\n");
+ return 0;
+}