You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by ga...@apache.org on 2017/06/16 07:19:12 UTC
avro git commit: AVRO-1985: Unreleased resources
Repository: avro
Updated Branches:
refs/heads/master edd8edda0 -> 528a1c8a9
AVRO-1985: Unreleased resources
This closes #177
Project: http://git-wip-us.apache.org/repos/asf/avro/repo
Commit: http://git-wip-us.apache.org/repos/asf/avro/commit/528a1c8a
Tree: http://git-wip-us.apache.org/repos/asf/avro/tree/528a1c8a
Diff: http://git-wip-us.apache.org/repos/asf/avro/diff/528a1c8a
Branch: refs/heads/master
Commit: 528a1c8a997a0831e83b525173c6a70b08ee681a
Parents: edd8edd
Author: Gabor Szadovszky <ga...@apache.org>
Authored: Thu Dec 29 12:27:38 2016 +0100
Committer: Gabor Szadovszky <ga...@apache.org>
Committed: Fri Jun 16 09:18:40 2017 +0200
----------------------------------------------------------------------
CHANGES.txt | 2 ++
.../main/java/org/apache/avro/data/Json.java | 9 ++++--
.../org/apache/avro/file/DataFileWriter.java | 14 +++++++--
.../test/java/org/apache/avro/TestDataFile.java | 32 +++++++++++---------
.../compiler/specific/SpecificCompiler.java | 19 +++++++-----
.../avro/mapreduce/AvroKeyOutputFormat.java | 12 ++++++--
.../mapreduce/AvroKeyValueOutputFormat.java | 13 ++++++--
.../main/java/org/apache/avro/tool/IdlTool.java | 9 ++++--
.../main/java/org/apache/avro/tool/Main.java | 14 +++++++--
9 files changed, 88 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/avro/blob/528a1c8a/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index c387de6..d7b0ec1 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -160,6 +160,8 @@ Trunk (not yet released)
AVRO-1401: @Nullable does not work with byte[] (Nandor Kollar via gabor)
+ AVRO-1985: Unreleased resources (gabor)
+
Avro 1.8.1 (14 May 2016)
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/avro/blob/528a1c8a/lang/java/avro/src/main/java/org/apache/avro/data/Json.java
----------------------------------------------------------------------
diff --git a/lang/java/avro/src/main/java/org/apache/avro/data/Json.java b/lang/java/avro/src/main/java/org/apache/avro/data/Json.java
index daa8482..66b92ef 100644
--- a/lang/java/avro/src/main/java/org/apache/avro/data/Json.java
+++ b/lang/java/avro/src/main/java/org/apache/avro/data/Json.java
@@ -18,6 +18,7 @@
package org.apache.avro.data;
import java.io.IOException;
+import java.io.InputStream;
import java.io.StringReader;
import java.util.Iterator;
@@ -55,8 +56,12 @@ public class Json {
public static final Schema SCHEMA;
static {
try {
- SCHEMA = Schema.parse
- (Json.class.getResourceAsStream("/org/apache/avro/data/Json.avsc"));
+ InputStream in = Json.class.getResourceAsStream("/org/apache/avro/data/Json.avsc");
+ try {
+ SCHEMA = Schema.parse(in);
+ } finally {
+ in.close();
+ }
} catch (IOException e) {
throw new AvroRuntimeException(e);
}
http://git-wip-us.apache.org/repos/asf/avro/blob/528a1c8a/lang/java/avro/src/main/java/org/apache/avro/file/DataFileWriter.java
----------------------------------------------------------------------
diff --git a/lang/java/avro/src/main/java/org/apache/avro/file/DataFileWriter.java b/lang/java/avro/src/main/java/org/apache/avro/file/DataFileWriter.java
index fe916dc..d70d4a4 100644
--- a/lang/java/avro/src/main/java/org/apache/avro/file/DataFileWriter.java
+++ b/lang/java/avro/src/main/java/org/apache/avro/file/DataFileWriter.java
@@ -190,11 +190,20 @@ public class DataFileWriter<D> implements Closeable, Flushable {
/** Open a writer appending to an existing file. */
public DataFileWriter<D> appendTo(File file) throws IOException {
- return appendTo(new SeekableFileInput(file),
- new SyncableFileOutputStream(file, true));
+ SeekableInput input = null;
+ try {
+ input = new SeekableFileInput(file);
+ OutputStream output = new SyncableFileOutputStream(file, true);
+ return appendTo(input, output);
+ } finally {
+ if (input != null)
+ input.close();
+ // output does not need to be closed here. It will be closed by invoking close() of this writer.
+ }
}
/** Open a writer appending to an existing file.
+ * <strong>Since 1.9.0 this method does not close in.</strong>
* @param in reading the existing file.
* @param out positioned at the end of the existing file.
*/
@@ -213,7 +222,6 @@ public class DataFileWriter<D> implements Closeable, Flushable {
} else {
this.codec = CodecFactory.nullCodec().createInstance();
}
- reader.close();
init(out);
http://git-wip-us.apache.org/repos/asf/avro/blob/528a1c8a/lang/java/avro/src/test/java/org/apache/avro/TestDataFile.java
----------------------------------------------------------------------
diff --git a/lang/java/avro/src/test/java/org/apache/avro/TestDataFile.java b/lang/java/avro/src/test/java/org/apache/avro/TestDataFile.java
index e067496..608d960 100644
--- a/lang/java/avro/src/test/java/org/apache/avro/TestDataFile.java
+++ b/lang/java/avro/src/test/java/org/apache/avro/TestDataFile.java
@@ -324,19 +324,23 @@ public class TestDataFile {
private void testFSync(boolean useFile) throws IOException {
DataFileWriter<Object> writer =
new DataFileWriter<Object>(new GenericDatumWriter<Object>());
- writer.setFlushOnEveryBlock(false);
- TestingByteArrayOutputStream out = new TestingByteArrayOutputStream();
- if (useFile) {
- File f = makeFile();
- SeekableFileInput in = new SeekableFileInput(f);
- writer.appendTo(in, out);
- } else {
- writer.create(SCHEMA, out);
- }
- int currentCount = 0;
- int syncCounter = 0;
try {
- for (Object datum : new RandomData(SCHEMA, COUNT, SEED+1)) {
+ writer.setFlushOnEveryBlock(false);
+ TestingByteArrayOutputStream out = new TestingByteArrayOutputStream();
+ if (useFile) {
+ File f = makeFile();
+ SeekableFileInput in = new SeekableFileInput(f);
+ try {
+ writer.appendTo(in, out);
+ } finally {
+ in.close();
+ }
+ } else {
+ writer.create(SCHEMA, out);
+ }
+ int currentCount = 0;
+ int syncCounter = 0;
+ for (Object datum : new RandomData(SCHEMA, COUNT, SEED + 1)) {
currentCount++;
writer.append(datum);
if (currentCount % 10 == 0) {
@@ -344,11 +348,11 @@ public class TestDataFile {
syncCounter++;
}
}
+ System.out.println("Total number of syncs: " + out.syncCount);
+ Assert.assertEquals(syncCounter, out.syncCount);
} finally {
writer.close();
}
- System.out.println("Total number of syncs: " + out.syncCount);
- Assert.assertEquals(syncCounter, out.syncCount);
}
http://git-wip-us.apache.org/repos/asf/avro/blob/528a1c8a/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java
----------------------------------------------------------------------
diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java b/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java
index 2959493..7a6f5f1 100644
--- a/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java
+++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java
@@ -270,17 +270,22 @@ public class SpecificCompiler {
if (src != null && f.exists() && f.lastModified() >= src.lastModified())
return f; // already up to date: ignore
f.getParentFile().mkdirs();
- Writer fw;
- if (outputCharacterEncoding != null) {
- fw = new OutputStreamWriter(new FileOutputStream(f), outputCharacterEncoding);
- } else {
- fw = new FileWriter(f);
- }
+ Writer fw = null;
+ FileOutputStream fos = null;
try {
+ if (outputCharacterEncoding != null) {
+ fos = new FileOutputStream(f);
+ fw = new OutputStreamWriter(fos, outputCharacterEncoding);
+ } else {
+ fw = new FileWriter(f);
+ }
fw.write(FILE_HEADER);
fw.write(contents);
} finally {
- fw.close();
+ if (fw != null)
+ fw.close();
+ if (fos != null)
+ fos.close();
}
return f;
}
http://git-wip-us.apache.org/repos/asf/avro/blob/528a1c8a/lang/java/mapred/src/main/java/org/apache/avro/mapreduce/AvroKeyOutputFormat.java
----------------------------------------------------------------------
diff --git a/lang/java/mapred/src/main/java/org/apache/avro/mapreduce/AvroKeyOutputFormat.java b/lang/java/mapred/src/main/java/org/apache/avro/mapreduce/AvroKeyOutputFormat.java
index 4c86112..125833c 100644
--- a/lang/java/mapred/src/main/java/org/apache/avro/mapreduce/AvroKeyOutputFormat.java
+++ b/lang/java/mapred/src/main/java/org/apache/avro/mapreduce/AvroKeyOutputFormat.java
@@ -102,8 +102,14 @@ public class AvroKeyOutputFormat<T> extends AvroOutputFormatBase<AvroKey<T>, Nul
GenericData dataModel = AvroSerialization.createDataModel(conf);
- return mRecordWriterFactory.create
- (writerSchema, dataModel, getCompressionCodec(context),
- getAvroFileOutputStream(context), getSyncInterval(context));
+ OutputStream out = getAvroFileOutputStream(context);
+ try {
+ return mRecordWriterFactory.create
+ (writerSchema, dataModel, getCompressionCodec(context),
+ out, getSyncInterval(context));
+ } catch (IOException e) {
+ out.close();
+ throw e;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/avro/blob/528a1c8a/lang/java/mapred/src/main/java/org/apache/avro/mapreduce/AvroKeyValueOutputFormat.java
----------------------------------------------------------------------
diff --git a/lang/java/mapred/src/main/java/org/apache/avro/mapreduce/AvroKeyValueOutputFormat.java b/lang/java/mapred/src/main/java/org/apache/avro/mapreduce/AvroKeyValueOutputFormat.java
index b844abd..7fd5d38 100644
--- a/lang/java/mapred/src/main/java/org/apache/avro/mapreduce/AvroKeyValueOutputFormat.java
+++ b/lang/java/mapred/src/main/java/org/apache/avro/mapreduce/AvroKeyValueOutputFormat.java
@@ -19,6 +19,7 @@
package org.apache.avro.mapreduce;
import java.io.IOException;
+import java.io.OutputStream;
import org.apache.avro.generic.GenericData;
import org.apache.avro.hadoop.io.AvroDatumConverter;
@@ -59,8 +60,14 @@ public class AvroKeyValueOutputFormat<K, V> extends AvroOutputFormatBase<K, V> {
GenericData dataModel = AvroSerialization.createDataModel(conf);
- return new AvroKeyValueRecordWriter<K, V>(keyConverter, valueConverter,
- dataModel, getCompressionCodec(context), getAvroFileOutputStream(context),
- getSyncInterval(context));
+ OutputStream out = getAvroFileOutputStream(context);
+ try {
+ return new AvroKeyValueRecordWriter<K, V>(keyConverter, valueConverter,
+ dataModel, getCompressionCodec(context), out,
+ getSyncInterval(context));
+ } catch (IOException e) {
+ out.close();
+ throw e;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/avro/blob/528a1c8a/lang/java/tools/src/main/java/org/apache/avro/tool/IdlTool.java
----------------------------------------------------------------------
diff --git a/lang/java/tools/src/main/java/org/apache/avro/tool/IdlTool.java b/lang/java/tools/src/main/java/org/apache/avro/tool/IdlTool.java
index f3cfe24..4095740 100644
--- a/lang/java/tools/src/main/java/org/apache/avro/tool/IdlTool.java
+++ b/lang/java/tools/src/main/java/org/apache/avro/tool/IdlTool.java
@@ -58,12 +58,17 @@ public class IdlTool implements Tool {
parser = new Idl(in);
}
- if (args.size() == 2 && ! "-".equals(args.get(1))) {
+ if (args.size() == 2 && !"-".equals(args.get(1))) {
parseOut = new PrintStream(new FileOutputStream(args.get(1)));
}
Protocol p = parser.CompilationUnit();
- parseOut.print(p.toString(true));
+ try {
+ parseOut.print(p.toString(true));
+ } finally {
+ if (parseOut != out) // Close only the newly created FileOutputStream
+ parseOut.close();
+ }
return 0;
}
http://git-wip-us.apache.org/repos/asf/avro/blob/528a1c8a/lang/java/tools/src/main/java/org/apache/avro/tool/Main.java
----------------------------------------------------------------------
diff --git a/lang/java/tools/src/main/java/org/apache/avro/tool/Main.java b/lang/java/tools/src/main/java/org/apache/avro/tool/Main.java
index 93c827c..aa689ec 100644
--- a/lang/java/tools/src/main/java/org/apache/avro/tool/Main.java
+++ b/lang/java/tools/src/main/java/org/apache/avro/tool/Main.java
@@ -89,9 +89,19 @@ public class Main {
}
}
System.err.print("Version ");
- printStream(Main.class.getClassLoader().getResourceAsStream("VERSION.txt"));
+ InputStream versionInput = Main.class.getClassLoader().getResourceAsStream("VERSION.txt");
+ try {
+ printStream(versionInput);
+ } finally {
+ versionInput.close();
+ }
System.err.print(" of ");
- printHead(Main.class.getClassLoader().getResourceAsStream("META-INF/NOTICE"), 5);
+ InputStream noticeInput = Main.class.getClassLoader().getResourceAsStream("META-INF/NOTICE");
+ try {
+ printHead(noticeInput, 5);
+ } finally {
+ noticeInput.close();
+ }
System.err.println("----------------");
System.err.println("Available tools:");