You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by ma...@apache.org on 2009/11/10 00:08:33 UTC
svn commit: r834276 - in /hadoop/avro/trunk: ./ src/c/ src/c/config/
src/test/schemata/
Author: massie
Date: Mon Nov 9 23:08:32 2009
New Revision: 834276
URL: http://svn.apache.org/viewvc?rev=834276&view=rev
Log:
AVRO-186. Full read-path interoperability test
Added:
hadoop/avro/trunk/src/c/config/
hadoop/avro/trunk/src/c/config/.gitignore
hadoop/avro/trunk/src/c/test_avro_data.c
- copied, changed from r833621, hadoop/avro/trunk/src/c/test_avro_primitives.c
Removed:
hadoop/avro/trunk/src/c/test_avro_primitives.c
Modified:
hadoop/avro/trunk/CHANGES.txt
hadoop/avro/trunk/src/c/Makefile.am
hadoop/avro/trunk/src/c/README
hadoop/avro/trunk/src/c/avro_array.c
hadoop/avro/trunk/src/c/avro_boolean.c
hadoop/avro/trunk/src/c/avro_bytes.c
hadoop/avro/trunk/src/c/avro_double.c
hadoop/avro/trunk/src/c/avro_enum.c
hadoop/avro/trunk/src/c/avro_fixed.c
hadoop/avro/trunk/src/c/avro_float.c
hadoop/avro/trunk/src/c/avro_int.c
hadoop/avro/trunk/src/c/avro_io_memory.c
hadoop/avro/trunk/src/c/avro_long.c
hadoop/avro/trunk/src/c/avro_map.c
hadoop/avro/trunk/src/c/avro_null.c
hadoop/avro/trunk/src/c/avro_record.c
hadoop/avro/trunk/src/c/avro_string.c
hadoop/avro/trunk/src/c/avro_union.c
hadoop/avro/trunk/src/c/avro_value.c
hadoop/avro/trunk/src/c/configure.in
hadoop/avro/trunk/src/c/json.c
hadoop/avro/trunk/src/c/json.h
hadoop/avro/trunk/src/c/test_avro_interop.c
hadoop/avro/trunk/src/test/schemata/interop.avsc
Modified: hadoop/avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/CHANGES.txt?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/CHANGES.txt (original)
+++ hadoop/avro/trunk/CHANGES.txt Mon Nov 9 23:08:32 2009
@@ -47,6 +47,8 @@
AVRO-177. Upgrade Java dependencies to recent versions. (cutting)
+ AVRO-186. Full read-path interoperability test (massie)
+
OPTIMIZATIONS
AVRO-172. More efficient schema processing (massie)
Modified: hadoop/avro/trunk/src/c/Makefile.am
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/Makefile.am?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/Makefile.am (original)
+++ hadoop/avro/trunk/src/c/Makefile.am Mon Nov 9 23:08:32 2009
@@ -23,7 +23,7 @@
-export-dynamic
-check_PROGRAMS=test_json_parser test_avro_schema test_avro_primitives
+check_PROGRAMS=test_json_parser test_avro_schema test_avro_data test_avro_interop
dist-hook: docs
@@ -35,13 +35,13 @@
test_avro_schema=test_avro_schema.c
test_avro_schema_LDADD=$(test_LDADD)
-test_avro_primitives=test_avro_primitives.c
-test_avro_primitives_LDADD=$(test_LDADD)
-# use -std=c99 to avoid warings about universal character names
-test_avro_primitives_CFLAGS=$(AM_CFLAGS) -std=c99
+test_avro_data=test_avro_data.c
+test_avro_data_LDADD=$(test_LDADD)
+# use -std=c99 to avoid warnings about universal character names
+test_avro_data_CFLAGS=$(AM_CFLAGS) -std=c99
-#test_avro_interop=test_avro_interop.c
-#test_avro_interop_LDADD=$(test_LDADD)
+test_avro_interop=test_avro_interop.c
+test_avro_interop_LDADD=$(test_LDADD)
TESTS=$(check_PROGRAMS)
Modified: hadoop/avro/trunk/src/c/README
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/README?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/README (original)
+++ hadoop/avro/trunk/src/c/README Mon Nov 9 23:08:32 2009
@@ -1,20 +1,14 @@
-Avro C Bindings (libavro)
+ ___ ______
+ / |_ ___________ / ____/
+ / /| | | / / ___/ __ \ / /
+ / ___ | |/ / / / /_/ / / /___
+/_/ |_|___/_/ \____/ \____/
-The code in this directory is the C bindings for Avro.
+======================================================
+Please see the documentation in the "docs" directory.
-Currently, there are only the Avro primitives with their corresponding
-unit tests. The unit tests currently only check for proper
-encoding/decoding within the C library. Interoperability tests are
-comming soon.
+Documentation can also be (re)generated by running...
-Only the memory-backed Avro handle is working right now although
-adding file and network io support is easy thank to the Apache
-Portable Runtime which the Avro C bindings depend on.
+$ make docs
-For API details, view the file avro.h. The documentation is a work in
-progress. The test cases (test_avro_*.c) also serve as great examples
-of how to use the API.
-libavro compile and passes all tests on Linux. On MacOS X, 4 out of 5
-unit tests pass. Portability is a definite design goal but currently
-the development emphasis is on Linux.
Modified: hadoop/avro/trunk/src/c/avro_array.c
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/avro_array.c?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/avro_array.c (original)
+++ hadoop/avro/trunk/src/c/avro_array.c Mon Nov 9 23:08:32 2009
@@ -21,13 +21,10 @@
struct avro_array_value
{
apr_array_header_t *values;
-
- struct avro_value *items;
+ const JSON_value *item_schema;
apr_pool_t *pool;
- int value_set;
-
/* Need the ctx to dupe value */
struct avro_value_ctx *ctx;
@@ -37,12 +34,21 @@
static void
avro_array_print (struct avro_value *value, FILE * fp)
{
+ int i;
struct avro_array_value *self =
container_of (value, struct avro_array_value, base_value);
avro_value_indent (value, fp);
- fprintf (fp, "array(%p) value items\n", self);
- avro_value_print_info (self->items, fp);
+ fprintf (fp, "array(%p) with %d items\n", self,
+ self->values ? self->values->nelts : 0);
+ if (self->values)
+ {
+ for (i = 0; i < self->values->nelts; i++)
+ {
+ avro_value *item = ((avro_value **) self->values->elts)[i];
+ avro_value_print_info (item, fp);
+ }
+ }
}
static avro_status_t
@@ -55,8 +61,6 @@
container_of (value, struct avro_array_value, base_value);
struct avro_io_reader *io;
- apr_pool_clear (self->pool);
-
if (!reader)
{
return AVRO_FAILURE;
@@ -76,9 +80,59 @@
if (count < 0)
{
/* TODO */
+ return AVRO_FAILURE;
}
- /* TODO */
+ /* Clear the pool to of any previous state */
+ apr_pool_clear (self->pool);
+
+ self->values =
+ skip ? NULL : apr_array_make (self->pool, count, sizeof (avro_value *));
+
+ while (count > 0)
+ {
+ /* Read in the count number of items */
+ if (skip)
+ {
+ avro_value *item =
+ avro_value_from_json (self->ctx, self->base_value.parent,
+ self->item_schema);
+ for (i = 0; i < count; i++)
+ {
+ status = avro_value_skip_data (item, reader);
+ if (status != AVRO_OK)
+ {
+ return status;
+ }
+ }
+ }
+ else
+ {
+ for (i = 0; i < count; i++)
+ {
+ avro_value *item =
+ avro_value_from_json (self->ctx, self->base_value.parent,
+ self->item_schema);
+ if (!item)
+ {
+ return AVRO_FAILURE;
+ }
+ status = avro_value_read_data (item, reader);
+ if (status != AVRO_OK)
+ {
+ return status;
+ }
+ /* Save the item */
+ *(avro_value **) apr_array_push (self->values) = item;
+ }
+ }
+
+ status = avro_read_long (io, &count);
+ if (status != AVRO_OK)
+ {
+ return status;
+ }
+ }
return AVRO_OK;
}
@@ -108,13 +162,15 @@
avro_array_create (struct avro_value_ctx *ctx, struct avro_value *parent,
apr_pool_t * pool, const JSON_value * json)
{
+ avro_value *value;
struct avro_array_value *self;
- const JSON_value *items;
+ apr_pool_t *tmp_pool;
self = apr_palloc (pool, sizeof (struct avro_array_value));
if (!self)
{
return NULL;
+ self->values = NULL;
}
self->base_value.type = AVRO_ARRAY;
self->base_value.pool = pool;
@@ -122,22 +178,31 @@
self->base_value.schema = json;
/* collect and save required items */
- items = json_attr_get (json, L"items");
- if (!items)
+ self->item_schema = json_attr_get (json, L"items");
+ if (!self->item_schema)
+ {
+ return NULL;
+ }
+
+ /* Validate the item schema */
+ if (apr_pool_create (&tmp_pool, pool) != APR_SUCCESS)
{
return NULL;
}
- self->items = avro_value_from_json (ctx, parent, items);
- if (!self->items)
+ value = avro_value_from_json (ctx, parent, self->item_schema);
+ apr_pool_clear (tmp_pool);
+ if (!value)
{
+ /* Invalid item schema */
return NULL;
}
+
/* Create a pool for the value processing */
if (apr_pool_create (&self->pool, pool) != APR_SUCCESS)
{
return NULL;
}
- self->value_set = 0;
+ self->values = NULL;
self->ctx = ctx;
return &self->base_value;
}
Modified: hadoop/avro/trunk/src/c/avro_boolean.c
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/avro_boolean.c?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/avro_boolean.c (original)
+++ hadoop/avro/trunk/src/c/avro_boolean.c Mon Nov 9 23:08:32 2009
@@ -31,7 +31,7 @@
struct avro_boolean_value *self =
container_of (value, struct avro_boolean_value, base_value);
avro_value_indent (value, fp);
- fprintf (fp, "boolean");
+ fprintf (fp, "boolean(%p)", self);
if (self->value_set)
{
fprintf (fp, " value=%d", self->value);
Modified: hadoop/avro/trunk/src/c/avro_bytes.c
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/avro_bytes.c?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/avro_bytes.c (original)
+++ hadoop/avro/trunk/src/c/avro_bytes.c Mon Nov 9 23:08:32 2009
@@ -35,7 +35,7 @@
container_of (value, struct avro_bytes_value, base_value);
avro_value_indent (value, fp);
- fprintf (fp, "bytes");
+ fprintf (fp, "bytes(%p)", self);
if (self->value_set)
{
fprintf (fp, " size=%ld ", self->size);
@@ -95,9 +95,8 @@
static avro_status_t
avro_bytes_write (struct avro_value *value, struct avro_writer *writer)
{
- struct avro_bytes_value *self =
- container_of (value, struct avro_bytes_value, base_value);
- return AVRO_OK;
+ /* TODO */
+ return AVRO_FAILURE;
}
static struct avro_value *
Modified: hadoop/avro/trunk/src/c/avro_double.c
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/avro_double.c?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/avro_double.c (original)
+++ hadoop/avro/trunk/src/c/avro_double.c Mon Nov 9 23:08:32 2009
@@ -31,7 +31,7 @@
struct avro_double_value *self =
container_of (value, struct avro_double_value, base_value);
avro_value_indent (value, fp);
- fprintf (fp, "double");
+ fprintf (fp, "double(%p)", self);
if (self->value_set)
{
fprintf (fp, " value=%f", self->value);
Modified: hadoop/avro/trunk/src/c/avro_enum.c
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/avro_enum.c?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/avro_enum.c (original)
+++ hadoop/avro/trunk/src/c/avro_enum.c Mon Nov 9 23:08:32 2009
@@ -33,7 +33,7 @@
struct avro_avro_enum *self =
container_of (value, struct avro_avro_enum, base_value);
avro_value_indent (value, fp);
- fprintf (fp, "enum name=%ls", self->name);
+ fprintf (fp, "enum(%p) name=%ls", self, self->name);
if (self->value_set)
{
fprintf (fp, " value=%ls",
@@ -127,7 +127,7 @@
/* register self with named objects */
apr_hash_set (ctx->named_objects, self->name,
- wcslen (self->name) * sizeof (wchar_t), &self->base_value);
+ wcslen (self->name) * sizeof (wchar_t), json);
/* collect and save required symbols */
symbols = json_attr_get_check_type (json, L"symbols", JSON_ARRAY);
Modified: hadoop/avro/trunk/src/c/avro_fixed.c
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/avro_fixed.c?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/avro_fixed.c (original)
+++ hadoop/avro/trunk/src/c/avro_fixed.c Mon Nov 9 23:08:32 2009
@@ -17,6 +17,7 @@
under the License.
*/
#include "avro_private.h"
+#include "dump.h"
struct avro_fixed_value
{
@@ -24,6 +25,8 @@
avro_long_t size;
void *value;
+ int value_set;
+
avro_value base_value;
};
@@ -33,31 +36,51 @@
struct avro_fixed_value *self =
container_of (value, struct avro_fixed_value, base_value);
avro_value_indent (value, fp);
- fprintf (fp, "fixed name=%ls size=%ld\n", self->name, self->size);
+ fprintf (fp, "fixed(%p) name=%ls size=%lld ", self, self->name, self->size);
+ if (self->value_set)
+ {
+ dump (fp, self->value, self->size);
+ }
}
static avro_status_t
-avro_fixed_read (struct avro_value *value, struct avro_reader *reader)
+avro_fixed_read_skip (struct avro_value *value, struct avro_reader *reader,
+ int skip)
{
struct avro_fixed_value *self =
container_of (value, struct avro_fixed_value, base_value);
- return AVRO_OK;
+ avro_io_reader *io;
+
+ if (!reader)
+ {
+ return AVRO_FAILURE;
+ }
+ io = reader->io;
+ if (!io)
+ {
+ return AVRO_FAILURE;
+ }
+ self->value_set = !skip;
+ return io->read (io, self->value, self->size);
+}
+
+static avro_status_t
+avro_fixed_read (struct avro_value *value, struct avro_reader *reader)
+{
+ return avro_fixed_read_skip (value, reader, 0);
}
static avro_status_t
avro_fixed_skip (struct avro_value *value, struct avro_reader *reader)
{
- struct avro_fixed_value *self =
- container_of (value, struct avro_fixed_value, base_value);
- return AVRO_OK;
+ return avro_fixed_read_skip (value, reader, 1);
}
static avro_status_t
avro_fixed_write (struct avro_value *value, struct avro_writer *writer)
{
- struct avro_fixed_value *self =
- container_of (value, struct avro_fixed_value, base_value);
- return AVRO_OK;
+ /* TODO */
+ return AVRO_FAILURE;
}
static struct avro_value *
@@ -86,7 +109,10 @@
return NULL;
}
self->size = size->json_number;
-
+ if (self->size < 0)
+ {
+ return NULL;
+ }
/* collect and save the require name */
name = json_attr_get_check_type (json, L"name", JSON_STRING);
@@ -97,7 +123,7 @@
self->name = name->json_string;
/* register self with named objects */
apr_hash_set (ctx->named_objects, self->name,
- wcslen (self->name) * sizeof (wchar_t), &self->base_value);
+ wcslen (self->name) * sizeof (wchar_t), json);
self->value = apr_palloc (pool, self->size);
if (!self->value)
@@ -105,6 +131,7 @@
return NULL;
}
+ self->value_set = 0;
return &self->base_value;
}
Modified: hadoop/avro/trunk/src/c/avro_float.c
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/avro_float.c?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/avro_float.c (original)
+++ hadoop/avro/trunk/src/c/avro_float.c Mon Nov 9 23:08:32 2009
@@ -31,7 +31,7 @@
struct avro_float_value *self =
container_of (value, struct avro_float_value, base_value);
avro_value_indent (value, fp);
- fprintf (fp, "float");
+ fprintf (fp, "float(%p)", self);
if (self->value_set)
{
fprintf (fp, " value=%f", self->value);
Modified: hadoop/avro/trunk/src/c/avro_int.c
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/avro_int.c?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/avro_int.c (original)
+++ hadoop/avro/trunk/src/c/avro_int.c Mon Nov 9 23:08:32 2009
@@ -31,7 +31,7 @@
struct avro_int_value *self =
container_of (value, struct avro_int_value, base_value);
avro_value_indent (value, fp);
- fprintf (fp, "int");
+ fprintf (fp, "int(%p)", self);
if (self->value_set)
{
fprintf (fp, " value=%d", self->value);
Modified: hadoop/avro/trunk/src/c/avro_io_memory.c
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/avro_io_memory.c?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/avro_io_memory.c (original)
+++ hadoop/avro/trunk/src/c/avro_io_memory.c Mon Nov 9 23:08:32 2009
@@ -80,7 +80,8 @@
self->used += len;
DEBUG (fprintf
(stderr, "avro_io_memory_write %d bytes, %d/%d used\n", len,
- self->used, self->len); dump (stderr, addr, len));
+ self->used, self->len);
+ dump (stderr, addr, len));
return AVRO_OK;
}
Modified: hadoop/avro/trunk/src/c/avro_long.c
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/avro_long.c?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/avro_long.c (original)
+++ hadoop/avro/trunk/src/c/avro_long.c Mon Nov 9 23:08:32 2009
@@ -31,7 +31,7 @@
struct avro_long_value *self =
container_of (value, struct avro_long_value, base_value);
avro_value_indent (value, fp);
- fprintf (fp, "long");
+ fprintf (fp, "long(%p)", self);
if (self->value_set)
{
fprintf (fp, " value=%ld", self->value);
Modified: hadoop/avro/trunk/src/c/avro_map.c
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/avro_map.c?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/avro_map.c (original)
+++ hadoop/avro/trunk/src/c/avro_map.c Mon Nov 9 23:08:32 2009
@@ -21,8 +21,13 @@
struct avro_map_value
{
apr_hash_t *map;
- struct avro_value *keys;
- struct avro_value *values;
+ apr_pool_t *pool;
+ const JSON_value *key_schema;
+ const JSON_value *value_schema;
+
+ /* Need the ctx to dupe value */
+ struct avro_value_ctx *ctx;
+
struct avro_value base_value;
};
@@ -31,34 +36,115 @@
{
struct avro_map_value *self =
container_of (value, struct avro_map_value, base_value);
+
avro_value_indent (value, fp);
- fprintf (fp, "map(%p) key/value\n", self);
- avro_value_print_info (self->keys, fp);
- avro_value_print_info (self->values, fp);
+ fprintf (fp, "map(%p)", self);
+ if (self->map)
+ {
+ int i;
+ apr_pool_t *subpool;
+ apr_hash_index_t *hi;
+ apr_ssize_t len;
+
+ fprintf (fp, " start\n");
+ apr_pool_create (&subpool, self->pool);
+ for (i = 0, hi = apr_hash_first (subpool, self->map); hi;
+ hi = apr_hash_next (hi), i++)
+ {
+ avro_value *key;
+ avro_value *value;
+ apr_hash_this (hi, (void *) &key, &len, (void *) &value);
+ avro_value_print_info (key, fp);
+ avro_value_print_info (value, fp);
+ }
+ apr_pool_clear (subpool);
+ avro_value_indent (value, fp);
+ fprintf (fp, "map(%p) end", self);
+ }
+ fprintf (fp, "\n");
}
static avro_status_t
-avro_map_read (struct avro_value *value, struct avro_reader *reader)
+avro_map_read_skip (struct avro_value *value, struct avro_reader *reader,
+ int skip)
{
struct avro_map_value *self =
container_of (value, struct avro_map_value, base_value);
+ avro_status_t status;
+ avro_long_t i, count;
+ struct avro_io_reader *io;
+
+ if (!reader)
+ {
+ return AVRO_FAILURE;
+ }
+ io = reader->io;
+ if (!io)
+ {
+ return AVRO_FAILURE;
+ }
+ status = avro_read_long (io, &count);
+ if (status != AVRO_OK)
+ {
+ return status;
+ }
+ apr_pool_clear (self->pool);
+ self->map = skip ? NULL : apr_hash_make (self->pool);
+
+ while (count > 0)
+ {
+ for (i = 0; i < count; i++)
+ {
+ avro_value *key =
+ avro_value_from_json (self->ctx, self->base_value.parent,
+ self->key_schema);
+ avro_value *value =
+ avro_value_from_json (self->ctx, self->base_value.parent,
+ self->value_schema);
+ if (!key || !value)
+ {
+ return AVRO_FAILURE;
+ }
+ status = avro_value_read_data (key, reader);
+ if (status != AVRO_OK)
+ {
+ return status;
+ }
+ status = avro_value_read_data (value, reader);
+ if (status != AVRO_OK)
+ {
+ return status;
+ }
+ apr_hash_set (self->map, key, sizeof (struct avro_value), value);
+ }
+
+ status = avro_read_long (io, &count);
+ if (status != AVRO_OK)
+ {
+ return status;
+ }
+ }
+
return AVRO_OK;
}
static avro_status_t
+avro_map_read (struct avro_value *value, struct avro_reader *reader)
+{
+ return avro_map_read_skip (value, reader, 0);
+}
+
+static avro_status_t
avro_map_skip (struct avro_value *value, struct avro_reader *reader)
{
- struct avro_map_value *self =
- container_of (value, struct avro_map_value, base_value);
- return AVRO_OK;
+ return avro_map_read_skip (value, reader, 1);
}
static avro_status_t
avro_map_write (struct avro_value *value, struct avro_writer *writer)
{
- struct avro_map_value *self =
- container_of (value, struct avro_map_value, base_value);
- return AVRO_OK;
+ /* TODO */
+ return AVRO_FAILURE;
}
static struct avro_value *
@@ -66,8 +152,8 @@
apr_pool_t * pool, const JSON_value * json)
{
struct avro_map_value *self;
- const JSON_value *keys;
- const JSON_value *values;
+ avro_value *value;
+ apr_pool_t *tmp_pool;
DEBUG (fprintf (stderr, "Creating map\n"));
self = apr_palloc (pool, sizeof (struct avro_map_value));
@@ -80,35 +166,36 @@
self->base_value.parent = parent;
self->base_value.schema = json;
- /* collect and save required keys */
- keys = json_attr_get (json, L"keys");
- if (keys)
- {
- self->keys = avro_value_from_json (ctx, &self->base_value, keys);
- if (!self->keys)
- {
- return NULL;
- }
- }
- else
+ /* map keys are always assumed to be strings */
+ char *default_key = "{\"type\":\"string\"}";
+ self->key_schema = JSON_parse (pool, default_key, strlen (default_key));
+ if (!self->key_schema)
{
- /* TODO: should keys default to string? */
- self->keys =
- avro_value_registry[AVRO_STRING]->create (ctx, &self->base_value,
- pool, NULL);
+ return NULL;
}
/* collect and save required values */
- values = json_attr_get (json, L"values");
- if (!values)
+ self->value_schema = json_attr_get (json, L"values");
+ if (!self->value_schema)
{
return NULL;
}
- self->values = avro_value_from_json (ctx, &self->base_value, values);
- if (!self->values)
+ /* verify that the value schema is valid */
+ apr_pool_create (&tmp_pool, pool);
+ value = avro_value_from_json (ctx, parent, self->value_schema);
+ apr_pool_clear (tmp_pool);
+ if (!value)
+ {
+ return NULL;
+ }
+
+ /* Create a value pool */
+ if (apr_pool_create (&self->pool, pool) != APR_SUCCESS)
{
return NULL;
}
+ self->map = NULL;
+ self->ctx = ctx;
return &self->base_value;
}
Modified: hadoop/avro/trunk/src/c/avro_null.c
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/avro_null.c?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/avro_null.c (original)
+++ hadoop/avro/trunk/src/c/avro_null.c Mon Nov 9 23:08:32 2009
@@ -34,7 +34,7 @@
avro_null_print (struct avro_value *value, FILE * fp)
{
avro_value_indent (value, fp);
- fprintf (fp, "null\n");
+ fprintf (fp, "null(%p)\n", value);
}
static struct avro_value *
Modified: hadoop/avro/trunk/src/c/avro_record.c
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/avro_record.c?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/avro_record.c (original)
+++ hadoop/avro/trunk/src/c/avro_record.c Mon Nov 9 23:08:32 2009
@@ -53,7 +53,7 @@
struct avro_record_value *self =
container_of (value, struct avro_record_value, base_value);
avro_value_indent (value, fp);
- fprintf (fp, "record name=%ls\n", self->name);
+ fprintf (fp, "record(%p) name=%ls\n", self, self->name);
for (i = 0; i < self->fields->nelts; i++)
{
struct avro_value *field =
@@ -182,10 +182,16 @@
static avro_status_t
avro_record_write (struct avro_value *value, struct avro_writer *writer)
{
-/* TODO:
- struct avro_record_value *record =
+ int i;
+ struct avro_record_value *self =
container_of (value, struct avro_record_value, base_value);
-*/
+
+ for (i = 0; i < self->fields->nelts; i++)
+ {
+ struct avro_value *field =
+ ((struct avro_value **) self->fields->elts)[i];
+ avro_value_write_data (field, writer);
+ }
return AVRO_OK;
}
@@ -223,7 +229,7 @@
/* register self with named objects */
apr_hash_set (ctx->named_objects, self->name,
- wcslen (self->name) * sizeof (wchar_t), &self->base_value);
+ wcslen (self->name) * sizeof (wchar_t), json);
/* collect and save optional namespace */
self->space = NULL;
Modified: hadoop/avro/trunk/src/c/avro_string.c
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/avro_string.c?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/avro_string.c (original)
+++ hadoop/avro/trunk/src/c/avro_string.c Mon Nov 9 23:08:32 2009
@@ -32,7 +32,7 @@
struct avro_string_value *self =
container_of (value, struct avro_string_value, base_value);
avro_value_indent (value, fp);
- fprintf (fp, "string");
+ fprintf (fp, "string(%p)", self);
if (self->value_set)
{
fprintf (fp, " value=%ls", self->value);
Modified: hadoop/avro/trunk/src/c/avro_union.c
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/avro_union.c?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/avro_union.c (original)
+++ hadoop/avro/trunk/src/c/avro_union.c Mon Nov 9 23:08:32 2009
@@ -21,6 +21,8 @@
struct avro_union_value
{
apr_array_header_t *schemas;
+ int value_set;
+ avro_long_t value_index;
struct avro_value base_value;
};
@@ -34,34 +36,78 @@
}
static avro_status_t
-avro_union_read (struct avro_value *value, struct avro_reader *reader)
+avro_union_read_skip (struct avro_value *value, struct avro_reader *reader,
+ int skip)
{
+ avro_status_t status;
struct avro_union_value *self =
container_of (value, struct avro_union_value, base_value);
- return AVRO_OK;
+ struct avro_io_reader *io;
+ avro_value *requested_value;
+
+ io = reader->io;
+ if (!io)
+ {
+ return AVRO_FAILURE;
+ }
+
+ status = avro_read_long (io, &self->value_index);
+ if (status != AVRO_OK)
+ {
+ return status;
+ }
+
+ if (self->value_index < 0 || self->value_index > self->schemas->nelts)
+ {
+ return AVRO_FAILURE;
+ }
+
+ requested_value = ((avro_value **) self->schemas->elts)[self->value_index];
+ if (!requested_value)
+ {
+ return AVRO_FAILURE;
+ }
+
+ self->value_set = !skip;
+
+ if (skip)
+ {
+ status = avro_value_skip_data (requested_value, reader);
+ }
+ else
+ {
+ status = avro_value_read_data (requested_value, reader);
+ }
+ return status;
+}
+
+static avro_status_t
+avro_union_read (struct avro_value *value, struct avro_reader *reader)
+{
+ return avro_union_read_skip (value, reader, 0);
}
static avro_status_t
avro_union_skip (struct avro_value *value, struct avro_reader *reader)
{
- struct avro_union_value *self =
- container_of (value, struct avro_union_value, base_value);
- return AVRO_OK;
+ return avro_union_read_skip (value, reader, 1);
}
static avro_status_t
avro_union_write (struct avro_value *value, struct avro_writer *writer)
{
- struct avro_union_value *self =
- container_of (value, struct avro_union_value, base_value);
- return AVRO_OK;
+ /* TODO */
+ return AVRO_FAILURE;
}
static struct avro_value *
avro_union_create (struct avro_value_ctx *ctx, struct avro_value *parent,
apr_pool_t * pool, const JSON_value * json)
{
+ avro_value *value;
struct avro_union_value *self;
+ int i;
+
DEBUG (fprintf (stderr, "Creating a union\n"));
if (json->type != JSON_ARRAY)
@@ -77,7 +123,26 @@
self->base_value.pool = pool;
self->base_value.parent = parent;
self->base_value.schema = json;
- /* TODO: check the schemas ... and save them */
+
+ self->schemas =
+ apr_array_make (pool, json->json_array->nelts, sizeof (avro_value *));
+ if (!self->schemas)
+ {
+ return NULL;
+ }
+ for (i = 0; i < json->json_array->nelts; i++)
+ {
+ value =
+ avro_value_from_json (ctx, self->base_value.parent,
+ ((JSON_value **) json->json_array->elts)[i]);
+ if (!value)
+ {
+ /* Invalid union schema */
+ return NULL;
+ }
+ *(avro_value **) apr_array_push (self->schemas) = value;
+ }
+ self->value_set = 0;
return &self->base_value;
}
Modified: hadoop/avro/trunk/src/c/avro_value.c
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/avro_value.c?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/avro_value.c (original)
+++ hadoop/avro/trunk/src/c/avro_value.c Mon Nov 9 23:08:32 2009
@@ -106,6 +106,7 @@
struct avro_decorator_value
{
+ struct avro_value_ctx *ctx;
struct avro_value *decoratee;
struct avro_value base_value;
};
@@ -116,37 +117,51 @@
struct avro_decorator_value *self =
container_of (value, struct avro_decorator_value, base_value);
avro_value_indent (value, fp);
- fprintf (fp, "avro decorator\n");
+ fprintf (fp, "decorator(%p)\n", self);
+ avro_value_print_info (self->decoratee, fp);
}
static avro_status_t
-avro_decorator_read (struct avro_value *value, struct avro_reader *reader)
+avro_decorator_read_skip (struct avro_value *value,
+ struct avro_reader *reader, int skip)
{
struct avro_decorator_value *self =
container_of (value, struct avro_decorator_value, base_value);
+ /* Create a new decoratee */
+ self->decoratee = avro_value_from_json (self->ctx, value, value->schema);
+ if (!self->decoratee)
+ {
+ return AVRO_FAILURE;
+ }
+ if (skip)
+ {
+ return avro_value_skip_data (self->decoratee, reader);
+ }
return avro_value_read_data (self->decoratee, reader);
}
static avro_status_t
+avro_decorator_read (struct avro_value *value, struct avro_reader *reader)
+{
+ return avro_decorator_read_skip (value, reader, 0);
+}
+
+static avro_status_t
avro_decorator_skip (struct avro_value *value, struct avro_reader *reader)
{
- struct avro_decorator_value *self =
- container_of (value, struct avro_decorator_value, base_value);
- return avro_value_skip_data (self->decoratee, reader);
+ return avro_decorator_read_skip (value, reader, 1);
}
static avro_status_t
avro_decorator_write (struct avro_value *value, struct avro_writer *writer)
{
- struct avro_decorator_value *self =
- container_of (value, struct avro_decorator_value, base_value);
- return avro_value_write_data (self->decoratee, writer);
+ return AVRO_FAILURE;
}
/* Used for recursive schemas */
struct avro_value *
avro_decorator_create (struct avro_value_ctx *ctx, struct avro_value *parent,
- apr_pool_t * pool, struct avro_value *decoratee)
+ apr_pool_t * pool, const JSON_value * json)
{
struct avro_decorator_value *self =
apr_palloc (pool, sizeof (struct avro_decorator_value));
@@ -155,28 +170,20 @@
{
return NULL;
}
+ /* Save away the context */
+ self->ctx = ctx;
self->base_value.type = AVRO_DECORATOR;
self->base_value.pool = pool;
self->base_value.parent = parent;
- self->base_value.schema = decoratee->schema;
- /* object we're decorating */
- self->decoratee = decoratee;
+ self->base_value.schema = json;
return &self->base_value;
}
-static struct avro_value *
-avro_decorator_create_noop (struct avro_value_ctx *ctx,
- struct avro_value *parent, apr_pool_t * pool,
- const JSON_value * json)
-{
- return NULL;
-}
-
const struct avro_value_info avro_decorator_info = {
.name = L"decorator",
.type = AVRO_DECORATOR,
.private = 1,
- .create = avro_decorator_create_noop,
+ .create = avro_decorator_create,
.formats = {{
.read_data = avro_decorator_read,
.skip_data = avro_decorator_skip,
@@ -198,6 +205,7 @@
apr_status_t status;
avro_type_t avro_type;
apr_pool_t *subpool;
+ const JSON_value *schema = json;
status = apr_pool_create (&subpool, parent ? parent->pool : NULL);
if (status != APR_SUCCESS)
@@ -205,25 +213,25 @@
return NULL;
}
- avro_status = avro_type_from_json (json, &avro_type);
+ avro_status = avro_type_from_json (schema, &avro_type);
if (avro_status != AVRO_OK)
{
- if (json->type == JSON_STRING)
+ if (!ctx || json->type != JSON_STRING)
{
- avro_value *named_object =
- apr_hash_get (ctx->named_objects, json->json_string,
- wcslen (json->json_string) * sizeof (wchar_t));
- if (named_object)
- {
- return avro_decorator_create (ctx, parent, subpool,
- named_object);
- }
+ return NULL;
}
- return NULL;
+ schema =
+ apr_hash_get (ctx->named_objects, json->json_string,
+ wcslen (json->json_string) * sizeof (wchar_t));
+ if (!schema)
+ {
+ return NULL;
+ }
+ avro_type = AVRO_DECORATOR;
}
- return avro_value_registry[avro_type]->private ? NULL :
- avro_value_registry[avro_type]->create (ctx, parent, subpool, json);
+ return avro_value_registry[avro_type]->create (ctx, parent, subpool,
+ schema);
}
struct avro_value *
Added: hadoop/avro/trunk/src/c/config/.gitignore
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/config/.gitignore?rev=834276&view=auto
==============================================================================
--- hadoop/avro/trunk/src/c/config/.gitignore (added)
+++ hadoop/avro/trunk/src/c/config/.gitignore Mon Nov 9 23:08:32 2009
@@ -0,0 +1 @@
+*
Modified: hadoop/avro/trunk/src/c/configure.in
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/configure.in?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/configure.in (original)
+++ hadoop/avro/trunk/src/c/configure.in Mon Nov 9 23:08:32 2009
@@ -78,12 +78,13 @@
AC_OUTPUT
echo
-echo "C bindings for"
-echo " _ "
-echo " / \__ ___ __ ___ "
-echo " / _ \ \ / / '__/ _ \ "
-echo " / ___ \ V /| | | (_) |"
-echo "/_/ \_\_/ |_| \___/ "
+echo
+# figlet -f slant "Avro C"
+echo " ___ ______"
+echo " / |_ ___________ / ____/"
+echo " / /| | | / / ___/ __ \ / / "
+echo " / ___ | |/ / / / /_/ / / /___ "
+echo "/_/ |_|___/_/ \____/ \____/ "
echo
echo "Version: $VERSION"
echo "Library: $LIBAVRO_VERSION"
Modified: hadoop/avro/trunk/src/c/json.c
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/json.c?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/json.c (original)
+++ hadoop/avro/trunk/src/c/json.c Mon Nov 9 23:08:32 2009
@@ -34,7 +34,7 @@
}
static void
-JSON_print_private (FILE * file, JSON_value * value, int *depth)
+JSON_print_private (FILE * file, const JSON_value * value, int *depth)
{
int i;
switch (value->type)
@@ -105,7 +105,7 @@
}
void
-JSON_print (FILE * file, JSON_value * value)
+JSON_print (FILE * file, const JSON_value * value)
{
int depth = 0;
if (!file || !value)
Modified: hadoop/avro/trunk/src/c/json.h
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/json.h?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/json.h (original)
+++ hadoop/avro/trunk/src/c/json.h Mon Nov 9 23:08:32 2009
@@ -61,7 +61,7 @@
JSON_value *JSON_value_new (apr_pool_t * pool, int type);
-void JSON_print (FILE * file, JSON_value * value);
+void JSON_print (FILE * file, const JSON_value * value);
struct JSON_ctx
{
Copied: hadoop/avro/trunk/src/c/test_avro_data.c (from r833621, hadoop/avro/trunk/src/c/test_avro_primitives.c)
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/test_avro_data.c?p2=hadoop/avro/trunk/src/c/test_avro_data.c&p1=hadoop/avro/trunk/src/c/test_avro_primitives.c&r1=833621&r2=834276&rev=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/test_avro_primitives.c (original)
+++ hadoop/avro/trunk/src/c/test_avro_data.c Mon Nov 9 23:08:32 2009
@@ -174,13 +174,14 @@
for (i = 0; i < num_rand_tests; i++)
{
avro_long_t long_in, long_out;
- avro_int_t int_in, int_out, *intp;
+ avro_int_t int_in, int_out;
rand_val = random ();
if (is_long)
{
- intp = (avro_int_t *) & long_in;
- intp[0] = random ();
- intp[1] = random ();
+ avro_long_t a, b;
+ a = random ();
+ b = random ();
+ long_in = (a << 32) | b;
}
else
{
@@ -349,6 +350,48 @@
return AVRO_OK;
}
+avro_status_t
+test_record (apr_pool_t * pool, avro_io_reader * reader,
+ avro_io_writer * writer)
+{
+ return AVRO_OK;
+}
+
+avro_status_t
+test_enum (apr_pool_t * pool, avro_io_reader * reader,
+ avro_io_writer * writer)
+{
+ return AVRO_OK;
+}
+
+avro_status_t
+test_array (apr_pool_t * pool, avro_io_reader * reader,
+ avro_io_writer * writer)
+{
+ return AVRO_OK;
+}
+
+avro_status_t
+test_map (apr_pool_t * pool, avro_io_reader * reader, avro_io_writer * writer)
+{
+ return AVRO_OK;
+}
+
+avro_status_t
+test_union (apr_pool_t * pool, avro_io_reader * reader,
+ avro_io_writer * writer)
+{
+ return AVRO_OK;
+}
+
+avro_status_t
+test_fixed (apr_pool_t * pool, avro_io_reader * reader,
+ avro_io_writer * writer)
+{
+ return AVRO_OK;
+}
+
+
int
main (void)
{
@@ -378,7 +421,17 @@
{
"boolean", test_boolean},
{
- "null", test_null}
+ "null", test_null},
+ {
+ "record", test_record},
+ {
+ "enum", test_enum},
+ {
+ "array", test_array},
+ {
+ "map", test_map},
+ {
+ "fixed", test_fixed}
};
srandom (time (NULL));
Modified: hadoop/avro/trunk/src/c/test_avro_interop.c
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/c/test_avro_interop.c?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/c/test_avro_interop.c (original)
+++ hadoop/avro/trunk/src/c/test_avro_interop.c Mon Nov 9 23:08:32 2009
@@ -5,11 +5,6 @@
apr_pool_t *pool;
/*
-TODO:
-
-We still need to create a File Object container to finish this test.
-For now, we're only validating that we can parse the schema.
-
$ ant generate-test-data
needs to be called to make sure that test data is created.
@@ -25,6 +20,9 @@
apr_size_t jsonlen;
struct avro_value *value;
struct avro_reader *reader;
+ char *suffixes[] = { "py", "java" };
+ char *suffix;
+ int i;
if (!srcdir)
{
@@ -43,8 +41,10 @@
jsontext = avro_util_file_read_full (pool, path, &jsonlen);
if (!jsontext)
{
- fprintf (stderr, "Failed to parse the JSON in file %s\n", path);
- return EXIT_FAILURE;
+ fprintf (stderr,
+ "Couldn't find file: %s. Can't run interop test out of tree\n",
+ path);
+ return EXIT_SUCCESS;
}
value = avro_value_create (pool, jsontext, jsonlen);
if (!value)
@@ -53,23 +53,29 @@
return EXIT_FAILURE;
}
- path =
- apr_pstrcat (pool, srcdir, "/../../build/test/data-files/test.py.avro",
- NULL);
- if (!path)
- {
- return EXIT_FAILURE;
- }
- reader =
- avro_reader_file_container_create (pool, path, APR_READ, APR_OS_DEFAULT);
- if (!reader)
+ for (i = 0; i < sizeof (suffixes) / sizeof (suffixes[0]); i++)
{
- fprintf (stderr, "Failed to open data file %s\n", path);
- return EXIT_FAILURE;
- }
+ suffix = suffixes[i];
- avro_value_read_data (value, reader);
- avro_value_print_info (value, stderr);
+ path =
+ apr_pstrcat (pool, srcdir, "/../../build/test/data-files/test.",
+ suffix, ".avro", NULL);
+ if (!path)
+ {
+ return EXIT_FAILURE;
+ }
+ fprintf (stderr, "Running test on %s...\n", path);
+ reader =
+ avro_reader_file_container_create (pool, path, APR_READ,
+ APR_OS_DEFAULT);
+ if (!reader)
+ {
+ fprintf (stderr, "Failed to open data file %s\n", path);
+ return EXIT_FAILURE;
+ }
+ avro_value_read_data (value, reader);
+ avro_value_print_info (value, stderr);
+ }
apr_pool_destroy (pool);
return EXIT_SUCCESS;
}
Modified: hadoop/avro/trunk/src/test/schemata/interop.avsc
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/test/schemata/interop.avsc?rev=834276&r1=834275&r2=834276&view=diff
==============================================================================
--- hadoop/avro/trunk/src/test/schemata/interop.avsc (original)
+++ hadoop/avro/trunk/src/test/schemata/interop.avsc Mon Nov 9 23:08:32 2009
@@ -10,7 +10,7 @@
{"name": "nullField", "type": "null"},
{"name": "arrayField", "type": {"type": "array", "items": "double"}},
{"name": "mapField", "type":
- {"type": "map", "keys": "long", "values":
+ {"type": "map", "values":
{"type": "record", "name": "Foo",
"fields": [{"name": "label", "type": "string"}]}}},
{"name": "unionField", "type":