You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by dc...@apache.org on 2012/05/21 14:28:03 UTC
svn commit: r1340989 - in /avro/trunk: CHANGES.txt lang/c/src/datafile.c
lang/c/tests/CMakeLists.txt lang/c/tests/test_avro_1087.c
Author: dcreager
Date: Mon May 21 12:28:02 2012
New Revision: 1340989
URL: http://svn.apache.org/viewvc?rev=1340989&view=rev
Log:
AVRO-1087. C: avro_file_writer_open() and appending values works correctly.
The avro_file_writer_open() function wasn't correctly setting up the
avro_file_writer_t instance to allow you to append data to an existing
file. We now open up the file for read+write, read in the header, and
set up the writer's codec to match what's in the existing file. The
patch includes a test case.
Contributed by Pugachev Maxim.
Added:
avro/trunk/lang/c/tests/test_avro_1087.c
Modified:
avro/trunk/CHANGES.txt
avro/trunk/lang/c/src/datafile.c
avro/trunk/lang/c/tests/CMakeLists.txt
Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1340989&r1=1340988&r2=1340989&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Mon May 21 12:28:02 2012
@@ -113,6 +113,9 @@ Avro 1.7.0 (unreleased)
AVRO-1091. C: Helper scripts for calling CMake.
(Vivek Nadkarni via dcreager)
+ AVRO-1087. C: avro_file_writer_open() and appending Avro values
+ works correctly. (Pugachev Maxim via dcreager)
+
Avro 1.6.3 (5 March 2012)
AVRO-1077. Missing 'inline' for union set function. (thiru)
Modified: avro/trunk/lang/c/src/datafile.c
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c/src/datafile.c?rev=1340989&r1=1340988&r2=1340989&view=diff
==============================================================================
--- avro/trunk/lang/c/src/datafile.c (original)
+++ avro/trunk/lang/c/src/datafile.c Mon May 21 12:28:02 2012
@@ -310,13 +310,17 @@ static int file_writer_open(const char *
FILE *fp;
avro_reader_t reader;
- fp = fopen(path, "rb");
+ /* Open for read AND write */
+ fp = fopen(path, "r+b");
if (!fp) {
avro_set_error("Error opening file: %s",
strerror(errno));
return errno;
}
- reader = avro_reader_file(fp);
+
+ /* Don`t close the underlying file descriptor, logrotate can
+ * vanish it from sight. */
+ reader = avro_reader_file_fp(fp, 0);
if (!reader) {
fclose(fp);
avro_set_error("Cannot create file reader for %s", path);
@@ -325,13 +329,44 @@ static int file_writer_open(const char *
rval =
file_read_header(reader, &w->writers_schema, w->codec, w->sync,
sizeof(w->sync));
+
avro_reader_free(reader);
- /* Position to end of file and get ready to write */
- rval = file_writer_init_fp(path, "ab", w);
if (rval) {
- avro_freet(struct avro_file_writer_t_, w);
+ fclose(fp);
+ return rval;
}
- return rval;
+
+ w->block_count = 0;
+
+ /* Position to end of file and get ready to write */
+ fseek(fp, 0, SEEK_END);
+
+ w->writer = avro_writer_file(fp);
+ if (!w->writer) {
+ fclose(fp);
+ avro_set_error("Cannot create file writer for %s", path);
+ return ENOMEM;
+ }
+
+ w->datum_buffer_size = DEFAULT_BLOCK_SIZE;
+ w->datum_buffer = avro_malloc(w->datum_buffer_size);
+
+ if(!w->datum_buffer) {
+ avro_set_error("Could not allocate datum buffer\n");
+ avro_writer_free(w->writer);
+ return ENOMEM;
+ }
+
+ w->datum_writer =
+ avro_writer_memory(w->datum_buffer, w->datum_buffer_size);
+ if (!w->datum_writer) {
+ avro_set_error("Cannot create datum writer for file %s", path);
+ avro_writer_free(w->writer);
+ avro_free(w->datum_buffer, w->datum_buffer_size);
+ return ENOMEM;
+ }
+
+ return 0;
}
int avro_file_writer_open(const char *path, avro_file_writer_t * writer)
@@ -346,8 +381,16 @@ int avro_file_writer_open(const char *pa
avro_set_error("Cannot create new file writer for %s", path);
return ENOMEM;
}
+ w->codec = avro_new(struct avro_codec_t_);
+ if (!w->codec) {
+ avro_set_error("Cannot allocate new codec");
+ avro_freet(struct avro_file_writer_t_, w);
+ return ENOMEM;
+ }
rval = file_writer_open(path, w);
if (rval) {
+ avro_codec_reset(w->codec);
+ avro_freet(struct avro_codec_t_, w->codec);
avro_freet(struct avro_file_writer_t_, w);
return rval;
}
Modified: avro/trunk/lang/c/tests/CMakeLists.txt
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c/tests/CMakeLists.txt?rev=1340989&r1=1340988&r2=1340989&view=diff
==============================================================================
--- avro/trunk/lang/c/tests/CMakeLists.txt (original)
+++ avro/trunk/lang/c/tests/CMakeLists.txt Mon May 21 12:28:02 2012
@@ -52,6 +52,7 @@ add_avro_test(test_avro_968)
add_avro_test(test_avro_984)
add_avro_test(test_avro_1034)
add_avro_test(test_avro_1084)
+add_avro_test(test_avro_1087)
add_avro_test(test_avro_data)
add_avro_test(test_refcount)
add_avro_test(test_cpp test_cpp.cpp)
Added: avro/trunk/lang/c/tests/test_avro_1087.c
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c/tests/test_avro_1087.c?rev=1340989&view=auto
==============================================================================
--- avro/trunk/lang/c/tests/test_avro_1087.c (added)
+++ avro/trunk/lang/c/tests/test_avro_1087.c Mon May 21 12:28:02 2012
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+#include <avro.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+const char PERSON_SCHEMA[] =
+"{\"type\":\"record\",\
+ \"name\":\"Person\",\
+ \"fields\":[\
+ {\"name\": \"ID\", \"type\": \"int\"}]}";
+
+const char *dbname = "test.db";
+avro_schema_t schema;
+
+void add_record (avro_file_writer_t writer)
+{
+ avro_datum_t main_datum = avro_record(schema);
+ avro_datum_t id_datum = avro_int32(1);
+
+ if (avro_record_set (main_datum, "ID", id_datum))
+ {
+ printf ("Unable to create datum");
+ exit (EXIT_FAILURE);
+ }
+
+ avro_file_writer_append (writer, main_datum);
+
+ avro_datum_decref (id_datum);
+ avro_datum_decref (main_datum);
+}
+
+void create_database()
+{
+ avro_file_writer_t writer;
+
+ if (avro_schema_from_json_literal (PERSON_SCHEMA, &schema))
+ {
+ printf ("Unable to parse schema\n");
+ exit (EXIT_FAILURE);
+ }
+
+ if (avro_file_writer_create ("test.db", schema, &writer))
+ {
+ printf ("There was an error creating db: %s\n", avro_strerror());
+ exit (EXIT_FAILURE);
+ }
+
+ add_record (writer);
+
+ avro_file_writer_flush (writer);
+ avro_file_writer_close (writer);
+}
+
+
+int main()
+{
+ avro_file_writer_t writer;
+
+ create_database();
+
+ avro_file_writer_open (dbname, &writer);
+ add_record (writer);
+
+ avro_file_writer_flush (writer);
+ avro_file_writer_close (writer);
+
+ remove (dbname);
+
+ return EXIT_SUCCESS;
+}