You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@avro.apache.org by Mahesh V <ma...@gmail.com> on 2013/09/24 17:33:12 UTC
avro accessing record within an array in C
Hi,
given a schema as shown below, how can I set values
to the record "array_record_1" in the array below.?
Any help will be appreciated.
thanks
Mahesh
avro_schema_t array_schema = avro_schema_get_subschema(sw_schema,
"some_array");
array_datum = avro_array(array_schema);
^^^^^ this works fine.
avro_schema_t array_rec_schema = avro_schema_get_subschema(sw_schema,
"array_record_1");
array_rec_datum = avro_record(array_rec_schema);
^^^^ this fails
char *some_schema =
"{"
"\"type\":\"record\", \"name\" : \"some_schema\", \"fields\": [ "
"{ \"name\" : \"some_var_a\" , \"type\" : \"int\" },"
"{"
"\"name\" : \"some_ip\", \"type\" : {"
"\"type\":\"record\", \"name\" : \"some_ip\", \"fields\" : [ "
" { \"name\" : \"ip_version\", \"type\" : \"int\" },"
" { \"name\" : \"ip_v4\", \"type\" : \"long\" },"
" { \"name\" : \"ip_v6\", \"type\" : \"string\" }"
"]}},"
"{ \"name\" : \"some_array\" , \"type\" : "
" { \"type\":\"array\", \"items\": "
"{"
"\"type\":\"record\", \"name\" : \"array_record_1\", \"fields\": [ "
"{ \"name\" : \"array_record_var_1\" , \"type\" : \"int\" },"
"{ \"name\" : \"array_record_var_2\" , \"type\" : \"boolean\" } "
"]"
"}"
Re: avro accessing record within an array in C
Posted by Mahesh V <ma...@gmail.com>.
Thanks a ton Mika.
I was able to loop through and get my code correct.
below is the snippet
So today I can do this
1) Given an XML - convert it to json and further to avro json using a
python script
2) fill the avro contents using a C program and serialize it
3) de-serialise it using either a C or python program.
e.g.
{
const char* name = 0;
int field_index= 0;
int i=0, j=0, k=0;
size_t field_count = 0;
size_t array_count = 0;
size_t rec_count = 0;
avro_value_t field, subfield, rec, subrec, array, subarray;
avro_type_t type;
avro_value_get_size(&value, &field_count);
printf("field count is %d \n", field_count);
for(i = 0; i < field_count; i++) {
avro_value_get_by_index(&value, i, &field, &name);
type = avro_value_get_type(&field);
printf(" field name is : %s , type =", name);
print_type(type);
if(type == AVRO_ARRAY ) {
avro_value_get_size(&field, &array_count);
printf(" array number of elements = %d \n", array_count);
for(j = 0; j < array_count; j++) {
avro_value_get_by_index(&field, j, &subfield, &name),
printf(" field name is : %s , type =", name);
type = avro_value_get_type(&subfield);
print_type(type);
printf("\n");
if(type == AVRO_RECORD) {
avro_value_get_size(&subfield, &rec_count);
printf(" record number of elements = %d \n",
rec_count);
for(k = 0; k < rec_count; k++) {
avro_value_get_by_index(&subfield, k, &subrec,
&name),
printf(" field name is : %s , type =", name);
type = avro_value_get_type(&subfield);
print_type(type);
printf("\n");
}
}
}
} else if(type == AVRO_RECORD) {
avro_value_get_size(&field, &rec_count);
printf(" record number of elements = %d \n", rec_count);
for(k = 0; k < rec_count; k++) {
avro_value_get_by_index(&field, k, &subrec, &name),
printf(" field name is : %s , type =", name);
type = avro_value_get_type(&field);
print_type(type);
printf("\n");
}
}
printf("\n");
}
}
On Wed, Sep 25, 2013 at 2:46 PM, Mika Ristimaki <mi...@gmail.com>wrote:
> Hi,
>
> I haven't tried that but it could be done something like this
>
> avro_schema_from_json(some_schema, strlen(some_schema), &schema, &error)
> iface = avro_generic_class_from_schema(schema);
> avro_generic_value_new(iface, &value);
>
> const char *field_name = avro_schema_record_field_name(&value, index);
>
> Then compare
>
> -Mika
>
> On Sep 25, 2013, at 11:18 AM, Mahesh V <ma...@gmail.com>
> wrote:
>
> > Thanks Mika,
> > It worked like a charm.
> >
> > I have another question.
> > If given a field name "ip_v4" how can I query the json to reach the
> right place without knowing the names?
> > i.e. I want to get each field by index and check its name to see if it
> matches the one provided.
> >
> > e.g. pseudo code
> > search_json(schema, name) {
> > val = get_value_by_index( 0)
> > if strcmp(val->name, name) {
> > I found my field;
> > }
> > }
> >
> > Is this possible at all?
> > thanks
> > Mahesh
> >
> >
> >
> >
> > On Wed, Sep 25, 2013 at 12:43 AM, Mika Ristimaki <
> mika.ristimaki@gmail.com> wrote:
> > Hi,
> >
> > I can't really help with the datum api since I haven't used it myself,
> but with newer avro_value api arrays should work like this:
> >
> > avro_value_iface_t *iface;
> > avro_value_t value, array, element, field;
> > avro_schema_t schema;
> > avro_schema_error_t error;
> >
> > avro_schema_from_json(some_schema, strlen(some_schema), &schema, &error);
> >
> > iface = avro_generic_class_from_schema(schema);
> > avro_generic_value_new(iface, &value);
> >
> > avro_value_get_by_name(&value, "some_array", &array, NULL);
> > avro_value_append(&array, &element,NULL);
> >
> > avro_value_get_by_name(&element, "array_record_var_1", &field, NULL);
> > avro_value_set_int(&field,123);
> >
> > avro_value_get_by_name(&element, "array_record_var_2", &field, NULL);
> > avro_value_set_boolean(&field,1);
> >
> > etc.
> >
> > I didn't try this at all, so there'll be typos and other errors, but
> hope it helps to find the solution….
> >
> > -Mika
> >
> >
> > On Sep 24, 2013, at 6:33 PM, Mahesh V <ma...@gmail.com>
> wrote:
> >
> > > Hi,
> > >
> > > given a schema as shown below, how can I set values
> > > to the record "array_record_1" in the array below.?
> > >
> > > Any help will be appreciated.
> > > thanks
> > > Mahesh
> > >
> > > avro_schema_t array_schema = avro_schema_get_subschema(sw_schema,
> "some_array");
> > > array_datum = avro_array(array_schema);
> > > ^^^^^ this works fine.
> > >
> > > avro_schema_t array_rec_schema = avro_schema_get_subschema(sw_schema,
> "array_record_1");
> > > array_rec_datum = avro_record(array_rec_schema);
> > > ^^^^ this fails
> > >
> > > char *some_schema =
> > > "{"
> > > "\"type\":\"record\", \"name\" : \"some_schema\", \"fields\": [ "
> > > "{ \"name\" : \"some_var_a\" , \"type\" : \"int\" },"
> > > "{"
> > > "\"name\" : \"some_ip\", \"type\" : {"
> > > "\"type\":\"record\", \"name\" : \"some_ip\", \"fields\" : [ "
> > > " { \"name\" : \"ip_version\", \"type\" : \"int\" },"
> > > " { \"name\" : \"ip_v4\", \"type\" : \"long\" },"
> > > " { \"name\" : \"ip_v6\", \"type\" : \"string\" }"
> > > "]}},"
> > > "{ \"name\" : \"some_array\" , \"type\" : "
> > > " { \"type\":\"array\", \"items\": "
> > > "{"
> > > "\"type\":\"record\", \"name\" : \"array_record_1\", \"fields\": [ "
> > > "{ \"name\" : \"array_record_var_1\" , \"type\" : \"int\" },"
> > > "{ \"name\" : \"array_record_var_2\" , \"type\" : \"boolean\" } "
> > > "]"
> > > "}"
> >
> >
>
>
Re: avro accessing record within an array in C
Posted by Mika Ristimaki <mi...@gmail.com>.
Hi,
I don't quite understand what are you trying to do. If you need to read items from an array and the array size is 0, there isn't anything to read. No need to get schema either in my opinion. If you really need to check if a field exists in schema, you should be able to use this
if (avro_schema_record_field_get_index(schema, "avro_record_var_1") == -1)
// No field called avro_record_var_1
You might first need to get subschemas or something…
-Mika
On Sep 25, 2013, at 9:20 PM, Mahesh V <ma...@gmail.com> wrote:
> Quick question.
> How do I read the schema of an empty array?
>
> for e.g if my schema look like whats shown below, then given a fieldname e.g. array_record_var_1,
> how do I check if the field exists inside the array , even if the array is empty.
>
> the below snippet of code works fine if the array has at least one element.
> if the array is not populated, the avro_value_get_size returns 0 and the code does not enter the for loop
>
> if(type == AVRO_ARRAY ) {
> avro_value_get_size(&field, &array_count);
> printf(" array number of elements = %d \n", array_count);
> for(j = 0; j < array_count; j++) {
> avro_value_get_by_index(&field, j, &subfield, &name),
> printf(" field name is : %s , type =", name);
> type = avro_value_get_type(&subfield);
> print_type(type);
> printf("\n");
> if(type == AVRO_RECORD) {
> avro_value_get_size(&subfield, &rec_count);
> printf(" record number of elements = %d \n", rec_count);
> for(k = 0; k < rec_count; k++) {
> avro_value_get_by_index(&subfield, k, &subrec, &name),
> printf(" field name is : %s , type =", name);
> type = avro_value_get_type(&subrec);
> print_type(type);
> printf("\n");
> }
> }
> }
> }
>
> > > char *some_schema =
> > > "{"
> > > "\"type\":\"record\", \"name\" : \"some_schema\", \"fields\": [ "
> > > "{ \"name\" : \"some_var_a\" , \"type\" : \"int\" },"
> > > "{"
> > > "\"name\" : \"some_ip\", \"type\" : {"
> > > "\"type\":\"record\", \"name\" : \"some_ip\", \"fields\" : [ "
> > > " { \"name\" : \"ip_version\", \"type\" : \"int\" },"
> > > " { \"name\" : \"ip_v4\", \"type\" : \"long\" },"
> > > " { \"name\" : \"ip_v6\", \"type\" : \"string\" }"
> > > "]}},"
> > > "{ \"name\" : \"some_array\" , \"type\" : "
> > > " { \"type\":\"array\", \"items\": "
> > > "{"
> > > "\"type\":\"record\", \"name\" : \"array_record_1\", \"fields\": [ "
> > > "{ \"name\" : \"array_record_var_1\" , \"type\" : \"int\" },"
> > > "{ \"name\" : \"array_record_var_2\" , \"type\" : \"boolean\" } "
> > > "]"
> > > "}"
>
>
>
>
Re: avro accessing record within an array in C
Posted by Mahesh V <ma...@gmail.com>.
Quick question.
How do I read the schema of an empty array?
for e.g if my schema look like whats shown below, then given a fieldname
e.g. array_record_var_1,
how do I check if the field exists inside the array , even if the array is
empty.
the below snippet of code works fine if the array has at least one element.
if the array is not populated, the avro_value_get_size returns 0 and the
code does not enter the for loop
if(type == AVRO_ARRAY ) {
avro_value_get_size(&field, &array_count);
printf(" array number of elements = %d \n", array_count);
for(j = 0; j < array_count; j++) {
avro_value_get_by_index(&field, j, &subfield, &name),
printf(" field name is : %s , type =", name);
type = avro_value_get_type(&subfield);
print_type(type);
printf("\n");
if(type == AVRO_RECORD) {
avro_value_get_size(&subfield, &rec_count);
printf(" record number of elements = %d \n",
rec_count);
for(k = 0; k < rec_count; k++) {
avro_value_get_by_index(&subfield, k, &subrec,
&name),
printf(" field name is : %s , type =", name);
type = avro_value_get_type(&subrec);
print_type(type);
printf("\n");
}
}
}
}
> > char *some_schema =
> > "{"
> > "\"type\":\"record\", \"name\" : \"some_schema\", \"fields\": [ "
> > "{ \"name\" : \"some_var_a\" , \"type\" : \"int\" },"
> > "{"
> > "\"name\" : \"some_ip\", \"type\" : {"
> > "\"type\":\"record\", \"name\" : \"some_ip\", \"fields\" : [ "
> > " { \"name\" : \"ip_version\", \"type\" : \"int\" },"
> > " { \"name\" : \"ip_v4\", \"type\" : \"long\" },"
> > " { \"name\" : \"ip_v6\", \"type\" : \"string\" }"
> > "]}},"
> > "{ \"name\" : \"some_array\" , \"type\" : "
> > " { \"type\":\"array\", \"items\": "
> > "{"
> > "\"type\":\"record\", \"name\" : \"array_record_1\", \"fields\": [ "
> > "{ \"name\" : \"array_record_var_1\" , \"type\" : \"int\" },"
> > "{ \"name\" : \"array_record_var_2\" , \"type\" : \"boolean\" } "
> > "]"
> > "}"
Re: avro accessing record within an array in C
Posted by Mika Ristimaki <mi...@gmail.com>.
Hi,
I haven't tried that but it could be done something like this
avro_schema_from_json(some_schema, strlen(some_schema), &schema, &error)
iface = avro_generic_class_from_schema(schema);
avro_generic_value_new(iface, &value);
const char *field_name = avro_schema_record_field_name(&value, index);
Then compare
-Mika
On Sep 25, 2013, at 11:18 AM, Mahesh V <ma...@gmail.com> wrote:
> Thanks Mika,
> It worked like a charm.
>
> I have another question.
> If given a field name "ip_v4" how can I query the json to reach the right place without knowing the names?
> i.e. I want to get each field by index and check its name to see if it matches the one provided.
>
> e.g. pseudo code
> search_json(schema, name) {
> val = get_value_by_index( 0)
> if strcmp(val->name, name) {
> I found my field;
> }
> }
>
> Is this possible at all?
> thanks
> Mahesh
>
>
>
>
> On Wed, Sep 25, 2013 at 12:43 AM, Mika Ristimaki <mi...@gmail.com> wrote:
> Hi,
>
> I can't really help with the datum api since I haven't used it myself, but with newer avro_value api arrays should work like this:
>
> avro_value_iface_t *iface;
> avro_value_t value, array, element, field;
> avro_schema_t schema;
> avro_schema_error_t error;
>
> avro_schema_from_json(some_schema, strlen(some_schema), &schema, &error);
>
> iface = avro_generic_class_from_schema(schema);
> avro_generic_value_new(iface, &value);
>
> avro_value_get_by_name(&value, "some_array", &array, NULL);
> avro_value_append(&array, &element,NULL);
>
> avro_value_get_by_name(&element, "array_record_var_1", &field, NULL);
> avro_value_set_int(&field,123);
>
> avro_value_get_by_name(&element, "array_record_var_2", &field, NULL);
> avro_value_set_boolean(&field,1);
>
> etc.
>
> I didn't try this at all, so there'll be typos and other errors, but hope it helps to find the solution….
>
> -Mika
>
>
> On Sep 24, 2013, at 6:33 PM, Mahesh V <ma...@gmail.com> wrote:
>
> > Hi,
> >
> > given a schema as shown below, how can I set values
> > to the record "array_record_1" in the array below.?
> >
> > Any help will be appreciated.
> > thanks
> > Mahesh
> >
> > avro_schema_t array_schema = avro_schema_get_subschema(sw_schema, "some_array");
> > array_datum = avro_array(array_schema);
> > ^^^^^ this works fine.
> >
> > avro_schema_t array_rec_schema = avro_schema_get_subschema(sw_schema, "array_record_1");
> > array_rec_datum = avro_record(array_rec_schema);
> > ^^^^ this fails
> >
> > char *some_schema =
> > "{"
> > "\"type\":\"record\", \"name\" : \"some_schema\", \"fields\": [ "
> > "{ \"name\" : \"some_var_a\" , \"type\" : \"int\" },"
> > "{"
> > "\"name\" : \"some_ip\", \"type\" : {"
> > "\"type\":\"record\", \"name\" : \"some_ip\", \"fields\" : [ "
> > " { \"name\" : \"ip_version\", \"type\" : \"int\" },"
> > " { \"name\" : \"ip_v4\", \"type\" : \"long\" },"
> > " { \"name\" : \"ip_v6\", \"type\" : \"string\" }"
> > "]}},"
> > "{ \"name\" : \"some_array\" , \"type\" : "
> > " { \"type\":\"array\", \"items\": "
> > "{"
> > "\"type\":\"record\", \"name\" : \"array_record_1\", \"fields\": [ "
> > "{ \"name\" : \"array_record_var_1\" , \"type\" : \"int\" },"
> > "{ \"name\" : \"array_record_var_2\" , \"type\" : \"boolean\" } "
> > "]"
> > "}"
>
>
Re: avro accessing record within an array in C
Posted by Mahesh V <ma...@gmail.com>.
Thanks Mika,
It worked like a charm.
I have another question.
If given a field name "ip_v4" how can I query the json to reach the right
place without knowing the names?
i.e. I want to get each field by index and check its name to see if it
matches the one provided.
e.g. pseudo code
search_json(schema, name) {
val = get_value_by_index( 0)
if strcmp(val->name, name) {
I found my field;
}
}
Is this possible at all?
thanks
Mahesh
On Wed, Sep 25, 2013 at 12:43 AM, Mika Ristimaki
<mi...@gmail.com>wrote:
> Hi,
>
> I can't really help with the datum api since I haven't used it myself, but
> with newer avro_value api arrays should work like this:
>
> avro_value_iface_t *iface;
> avro_value_t value, array, element, field;
> avro_schema_t schema;
> avro_schema_error_t error;
>
> avro_schema_from_json(some_schema, strlen(some_schema), &schema, &error);
>
> iface = avro_generic_class_from_schema(schema);
> avro_generic_value_new(iface, &value);
>
> avro_value_get_by_name(&value, "some_array", &array, NULL);
> avro_value_append(&array, &element,NULL);
>
> avro_value_get_by_name(&element, "array_record_var_1", &field, NULL);
> avro_value_set_int(&field,123);
>
> avro_value_get_by_name(&element, "array_record_var_2", &field, NULL);
> avro_value_set_boolean(&field,1);
>
> etc.
>
> I didn't try this at all, so there'll be typos and other errors, but hope
> it helps to find the solution….
>
> -Mika
>
>
> On Sep 24, 2013, at 6:33 PM, Mahesh V <ma...@gmail.com>
> wrote:
>
> > Hi,
> >
> > given a schema as shown below, how can I set values
> > to the record "array_record_1" in the array below.?
> >
> > Any help will be appreciated.
> > thanks
> > Mahesh
> >
> > avro_schema_t array_schema = avro_schema_get_subschema(sw_schema,
> "some_array");
> > array_datum = avro_array(array_schema);
> > ^^^^^ this works fine.
> >
> > avro_schema_t array_rec_schema = avro_schema_get_subschema(sw_schema,
> "array_record_1");
> > array_rec_datum = avro_record(array_rec_schema);
> > ^^^^ this fails
> >
> > char *some_schema =
> > "{"
> > "\"type\":\"record\", \"name\" : \"some_schema\", \"fields\": [ "
> > "{ \"name\" : \"some_var_a\" , \"type\" : \"int\" },"
> > "{"
> > "\"name\" : \"some_ip\", \"type\" : {"
> > "\"type\":\"record\", \"name\" : \"some_ip\", \"fields\" : [ "
> > " { \"name\" : \"ip_version\", \"type\" : \"int\" },"
> > " { \"name\" : \"ip_v4\", \"type\" : \"long\" },"
> > " { \"name\" : \"ip_v6\", \"type\" : \"string\" }"
> > "]}},"
> > "{ \"name\" : \"some_array\" , \"type\" : "
> > " { \"type\":\"array\", \"items\": "
> > "{"
> > "\"type\":\"record\", \"name\" : \"array_record_1\", \"fields\": [ "
> > "{ \"name\" : \"array_record_var_1\" , \"type\" : \"int\" },"
> > "{ \"name\" : \"array_record_var_2\" , \"type\" : \"boolean\" } "
> > "]"
> > "}"
>
>
Re: avro accessing record within an array in C
Posted by Mika Ristimaki <mi...@gmail.com>.
Hi,
I can't really help with the datum api since I haven't used it myself, but with newer avro_value api arrays should work like this:
avro_value_iface_t *iface;
avro_value_t value, array, element, field;
avro_schema_t schema;
avro_schema_error_t error;
avro_schema_from_json(some_schema, strlen(some_schema), &schema, &error);
iface = avro_generic_class_from_schema(schema);
avro_generic_value_new(iface, &value);
avro_value_get_by_name(&value, "some_array", &array, NULL);
avro_value_append(&array, &element,NULL);
avro_value_get_by_name(&element, "array_record_var_1", &field, NULL);
avro_value_set_int(&field,123);
avro_value_get_by_name(&element, "array_record_var_2", &field, NULL);
avro_value_set_boolean(&field,1);
etc.
I didn't try this at all, so there'll be typos and other errors, but hope it helps to find the solution….
-Mika
On Sep 24, 2013, at 6:33 PM, Mahesh V <ma...@gmail.com> wrote:
> Hi,
>
> given a schema as shown below, how can I set values
> to the record "array_record_1" in the array below.?
>
> Any help will be appreciated.
> thanks
> Mahesh
>
> avro_schema_t array_schema = avro_schema_get_subschema(sw_schema, "some_array");
> array_datum = avro_array(array_schema);
> ^^^^^ this works fine.
>
> avro_schema_t array_rec_schema = avro_schema_get_subschema(sw_schema, "array_record_1");
> array_rec_datum = avro_record(array_rec_schema);
> ^^^^ this fails
>
> char *some_schema =
> "{"
> "\"type\":\"record\", \"name\" : \"some_schema\", \"fields\": [ "
> "{ \"name\" : \"some_var_a\" , \"type\" : \"int\" },"
> "{"
> "\"name\" : \"some_ip\", \"type\" : {"
> "\"type\":\"record\", \"name\" : \"some_ip\", \"fields\" : [ "
> " { \"name\" : \"ip_version\", \"type\" : \"int\" },"
> " { \"name\" : \"ip_v4\", \"type\" : \"long\" },"
> " { \"name\" : \"ip_v6\", \"type\" : \"string\" }"
> "]}},"
> "{ \"name\" : \"some_array\" , \"type\" : "
> " { \"type\":\"array\", \"items\": "
> "{"
> "\"type\":\"record\", \"name\" : \"array_record_1\", \"fields\": [ "
> "{ \"name\" : \"array_record_var_1\" , \"type\" : \"int\" },"
> "{ \"name\" : \"array_record_var_2\" , \"type\" : \"boolean\" } "
> "]"
> "}"