You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by ni...@apache.org on 2016/05/02 17:36:19 UTC
[02/12] avro git commit: AVRO-1828: Add EditorConfig file and cleanup
of whitespace violations
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java
----------------------------------------------------------------------
diff --git a/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java b/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java
index 9661e4e..0270b71 100644
--- a/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java
+++ b/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java
@@ -52,7 +52,7 @@ public class TestDataFileTools {
static String jsonData;
static Schema schema;
static File schemaFile;
-
+
private static final String KEY_NEEDING_ESCAPES = "trn\\\r\t\n";
private static final String ESCAPED_KEY = "trn\\\\\\r\\t\\n";
@@ -65,7 +65,7 @@ public class TestDataFileTools {
FileWriter fw = new FileWriter(schemaFile);
fw.append(schema.toString());
fw.close();
-
+
DataFileWriter<Object> writer
= new DataFileWriter<Object>(new GenericDatumWriter<Object>(schema))
.setMeta(KEY_NEEDING_ESCAPES, "")
@@ -80,10 +80,10 @@ public class TestDataFileTools {
writer.flush();
writer.close();
-
+
jsonData = builder.toString();
}
-
+
private String run(Tool tool, String... args) throws Exception {
return run(tool, null, args);
}
@@ -110,49 +110,49 @@ public class TestDataFileTools {
FileInputStream stdin = new FileInputStream(sampleFile);
assertEquals(jsonData, run(new DataFileReadTool(), stdin, "-"));
}
-
+
@Test
public void testReadToJsonPretty() throws Exception {
assertEquals(jsonData,
run(new DataFileReadTool(), "--pretty", sampleFile.getPath()));
}
-
+
@Test
public void testGetMeta() throws Exception {
String output = run(new DataFileGetMetaTool(), sampleFile.getPath());
assertTrue(output, output.contains("avro.schema\t"+schema.toString()+"\n"));
assertTrue(output, output.contains(ESCAPED_KEY+"\t\n"));
}
-
+
@Test
public void testGetMetaForSingleKey() throws Exception {
assertEquals(schema.toString() + "\n",
run(new DataFileGetMetaTool(), sampleFile.getPath(), "--key",
"avro.schema"));
}
-
+
@Test
public void testGetSchema() throws Exception {
assertEquals(schema.toString() + "\n",
run(new DataFileGetSchemaTool(), sampleFile.getPath()));
}
-
+
@Test
public void testWriteWithDeflate() throws Exception {
testWrite("deflate", Arrays.asList("--codec", "deflate"), "deflate");
}
-
+
@Test
public void testWrite() throws Exception {
testWrite("plain", Collections.<String>emptyList(), "null");
}
-
- public void testWrite(String name, List<String> extra, String expectedCodec)
+
+ public void testWrite(String name, List<String> extra, String expectedCodec)
throws Exception {
testWrite(name, extra, expectedCodec, "-schema", schema.toString());
testWrite(name, extra, expectedCodec, "-schema-file", schemaFile.toString());
}
- public void testWrite(String name, List<String> extra, String expectedCodec, String... extraArgs)
+ public void testWrite(String name, List<String> extra, String expectedCodec, String... extraArgs)
throws Exception {
File outFile = AvroTestUtil.tempFile(getClass(),
TestDataFileTools.class + ".testWrite." + name + ".avro");
@@ -171,7 +171,7 @@ public class TestDataFileTools {
args);
out.close();
fout.close();
-
+
// Read it back, and make sure it's valid.
GenericDatumReader<Object> reader = new GenericDatumReader<Object>();
DataFileReader<Object> fileReader = new DataFileReader<Object>(outFile,reader);
@@ -188,7 +188,7 @@ public class TestDataFileTools {
}
assertEquals(expectedCodec, codecStr);
}
-
+
@Test
public void testFailureOnWritingPartialJSONValues() throws Exception {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
@@ -197,7 +197,7 @@ public class TestDataFileTools {
new DataFileWriteTool().run(
new StringBufferInputStream("{"),
new PrintStream(out), // stdout
- null, // stderr
+ null, // stderr
Arrays.asList("-schema", "{ \"type\":\"record\", \"fields\":" +
"[{\"name\":\"foo\", \"type\":\"string\"}], " +
"\"name\":\"boring\" }", "-"));
@@ -206,7 +206,7 @@ public class TestDataFileTools {
// expected
}
}
-
+
@Test
public void testWritingZeroJsonValues() throws Exception {
File outFile = writeToAvroFile("zerojsonvalues",
@@ -214,10 +214,10 @@ public class TestDataFileTools {
"");
assertEquals(0, countRecords(outFile));
}
-
+
private int countRecords(File outFile) throws IOException {
GenericDatumReader<Object> reader = new GenericDatumReader<Object>();
- DataFileReader<Object> fileReader =
+ DataFileReader<Object> fileReader =
new DataFileReader<Object>(outFile,reader);
int i = 0;
for (@SuppressWarnings("unused") Object datum : fileReader) {
@@ -229,12 +229,12 @@ public class TestDataFileTools {
@Test
public void testDifferentSeparatorsBetweenJsonRecords() throws Exception {
File outFile = writeToAvroFile(
- "seperators",
- "{ \"type\":\"array\", \"items\":\"int\" }",
+ "seperators",
+ "{ \"type\":\"array\", \"items\":\"int\" }",
"[] [] []\n[][3] ");
assertEquals(5, countRecords(outFile));
}
-
+
public File writeToAvroFile(String testName, String schema, String json) throws Exception {
File outFile = AvroTestUtil.tempFile(getClass(),
TestDataFileTools.class + "." + testName + ".avro");
@@ -249,5 +249,5 @@ public class TestDataFileTools {
fout.close();
return outFile;
}
-
+
}
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/tools/src/test/java/org/apache/avro/tool/TestJsonToFromBinaryFragmentTools.java
----------------------------------------------------------------------
diff --git a/lang/java/tools/src/test/java/org/apache/avro/tool/TestJsonToFromBinaryFragmentTools.java b/lang/java/tools/src/test/java/org/apache/avro/tool/TestJsonToFromBinaryFragmentTools.java
index 557cac9..c5e3c97 100644
--- a/lang/java/tools/src/test/java/org/apache/avro/tool/TestJsonToFromBinaryFragmentTools.java
+++ b/lang/java/tools/src/test/java/org/apache/avro/tool/TestJsonToFromBinaryFragmentTools.java
@@ -37,22 +37,22 @@ import org.junit.BeforeClass;
import org.junit.Test;
/**
- * Tests both {@link JsonToBinaryFragmentTool}
+ * Tests both {@link JsonToBinaryFragmentTool}
* and {@link BinaryFragmentToJsonTool}.
*/
public class TestJsonToFromBinaryFragmentTools {
private static final String STRING_SCHEMA = Schema.create(Type.STRING).toString();
private static final String UTF8 = "utf-8";
- private static final String AVRO =
+ private static final String AVRO =
"ZLong string implies readable length encoding.";
- private static final String JSON =
+ private static final String JSON =
"\"Long string implies readable length encoding.\"\n";
@Test
public void testBinaryToJson() throws Exception {
binaryToJson(AVRO, JSON, STRING_SCHEMA);
}
-
+
@Test
public void testJsonToBinary() throws Exception {
jsonToBinary(JSON, AVRO, STRING_SCHEMA);
@@ -82,7 +82,7 @@ public class TestJsonToFromBinaryFragmentTools {
public void testBinaryToJsonSchemaFile() throws Exception {
binaryToJson(AVRO, JSON, "--schema-file", schemaFile());
}
-
+
@Test
public void testJsonToBinarySchemaFile() throws Exception {
jsonToBinary(JSON, AVRO, "--schema-file", schemaFile());
@@ -91,7 +91,7 @@ public class TestJsonToFromBinaryFragmentTools {
private void binaryToJson(String avro, String json, String... options) throws Exception {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintStream p = new PrintStream(new BufferedOutputStream(baos));
-
+
List<String> args = new ArrayList<String>();
args.addAll(Arrays.asList(options));
args.add("-");
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/tools/src/test/java/org/apache/avro/tool/TestRpcReceiveAndSendTools.java
----------------------------------------------------------------------
diff --git a/lang/java/tools/src/test/java/org/apache/avro/tool/TestRpcReceiveAndSendTools.java b/lang/java/tools/src/test/java/org/apache/avro/tool/TestRpcReceiveAndSendTools.java
index 97e527d..7e38198 100644
--- a/lang/java/tools/src/test/java/org/apache/avro/tool/TestRpcReceiveAndSendTools.java
+++ b/lang/java/tools/src/test/java/org/apache/avro/tool/TestRpcReceiveAndSendTools.java
@@ -27,7 +27,7 @@ import java.util.Arrays;
import org.junit.Test;
public class TestRpcReceiveAndSendTools {
-
+
/**
* Starts a server (using the tool) and sends a single message to it.
*/
@@ -38,7 +38,7 @@ public class TestRpcReceiveAndSendTools {
ByteArrayOutputStream baos1 = new ByteArrayOutputStream();
PrintStream p1 = new PrintStream(baos1);
RpcReceiveTool receive = new RpcReceiveTool();
- receive.run1(null, p1, System.err,
+ receive.run1(null, p1, System.err,
Arrays.asList("http://0.0.0.0:0/",
protocolFile, "hello",
"-data", "\"Hello!\""));
@@ -47,10 +47,10 @@ public class TestRpcReceiveAndSendTools {
RpcSendTool send = new RpcSendTool();
send.run(null, p2, System.err,
Arrays.asList("http://127.0.0.1:"+receive.server.getPort()+"/",
- protocolFile, "hello",
+ protocolFile, "hello",
"-data", "{ \"greeting\": \"Hi!\" }"));
receive.run2(System.err);
-
+
assertTrue(baos1.toString("UTF-8").replace("\r", "")
.endsWith("hello\t{\"greeting\":\"Hi!\"}\n"));
assertEquals("\"Hello!\"\n", baos2.toString("UTF-8").replace("\r", ""));
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/tools/src/test/java/org/apache/avro/tool/TestTextFileTools.java
----------------------------------------------------------------------
diff --git a/lang/java/tools/src/test/java/org/apache/avro/tool/TestTextFileTools.java b/lang/java/tools/src/test/java/org/apache/avro/tool/TestTextFileTools.java
index eee4027..0e3c5e8 100644
--- a/lang/java/tools/src/test/java/org/apache/avro/tool/TestTextFileTools.java
+++ b/lang/java/tools/src/test/java/org/apache/avro/tool/TestTextFileTools.java
@@ -57,7 +57,7 @@ public class TestTextFileTools {
static ByteBuffer[] lines;
static Schema schema;
static File schemaFile;
-
+
@BeforeClass
public static void writeRandomFile() throws IOException {
schema = Schema.create(Type.BYTES);
@@ -82,7 +82,7 @@ public class TestTextFileTools {
}
out.close();
}
-
+
private void fromText(String name, String... args) throws Exception {
File avroFile = AvroTestUtil.tempFile(getClass(), name + ".avro");
@@ -92,7 +92,7 @@ public class TestTextFileTools {
arglist.add(avroFile.toString());
new FromTextTool().run(null, null, null, arglist);
-
+
// Read it back, and make sure it's valid.
DataFileReader<ByteBuffer> file = new DataFileReader<ByteBuffer>
(avroFile, new GenericDatumReader<ByteBuffer>());
@@ -104,7 +104,7 @@ public class TestTextFileTools {
}
assertEquals(COUNT, i);
}
-
+
@Test
public void testFromText() throws Exception {
fromText("null", "--codec", "null");
@@ -128,7 +128,7 @@ public class TestTextFileTools {
arglist.add(outFile.toString());
new ToTextTool().run(null, null, null, arglist);
-
+
// Read it back, and make sure it's valid.
InputStream orig = new BufferedInputStream(new FileInputStream(linesFile));
InputStream after = new BufferedInputStream(new FileInputStream(outFile));
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/tools/src/test/java/org/apache/avro/tool/TestToTrevniTool.java
----------------------------------------------------------------------
diff --git a/lang/java/tools/src/test/java/org/apache/avro/tool/TestToTrevniTool.java b/lang/java/tools/src/test/java/org/apache/avro/tool/TestToTrevniTool.java
index 31f1ab6..164acaf 100644
--- a/lang/java/tools/src/test/java/org/apache/avro/tool/TestToTrevniTool.java
+++ b/lang/java/tools/src/test/java/org/apache/avro/tool/TestToTrevniTool.java
@@ -50,7 +50,7 @@ public class TestToTrevniTool {
new ToTrevniTool().run(null, p, null, Arrays.asList(args));
return baos.toString("UTF-8").replace("\r", "");
}
-
+
@Test
public void test() throws Exception {
Schema schema = Schema.parse(SCHEMA_FILE);
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnReader.java
----------------------------------------------------------------------
diff --git a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnReader.java b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnReader.java
index f7514db..97ee485 100644
--- a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnReader.java
+++ b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnReader.java
@@ -50,7 +50,7 @@ public class AvroColumnReader<D>
private GenericData model;
private Schema fileSchema;
private Schema readSchema;
-
+
private ColumnValues[] values;
private int[] arrayWidths;
private int column; // current index in values
@@ -125,16 +125,16 @@ public class AvroColumnReader<D>
switch (read.getType()) {
case NULL: case BOOLEAN:
case INT: case LONG:
- case FLOAT: case DOUBLE:
- case BYTES: case STRING:
+ case FLOAT: case DOUBLE:
+ case BYTES: case STRING:
case ENUM: case FIXED:
if (read.getType() != write.getType())
throw new TrevniRuntimeException("Type mismatch: "+read+" & "+write);
break;
- case MAP:
+ case MAP:
findDefaults(read.getValueType(), write.getValueType());
break;
- case ARRAY:
+ case ARRAY:
findDefaults(read.getElementType(), write.getElementType());
break;
case UNION:
@@ -145,7 +145,7 @@ public class AvroColumnReader<D>
findDefaults(s, write.getTypes().get(index));
}
break;
- case RECORD:
+ case RECORD:
for (Field f : read.getFields()) {
Field g = write.getField(f.name());
if (g == null)
@@ -200,7 +200,7 @@ public class AvroColumnReader<D>
final int startColumn = column;
switch (s.getType()) {
- case MAP:
+ case MAP:
int size = values[column].nextLength();
Map map = (Map)new HashMap(size);
for (int i = 0; i < size; i++) {
@@ -211,7 +211,7 @@ public class AvroColumnReader<D>
}
column = startColumn + arrayWidths[startColumn];
return map;
- case RECORD:
+ case RECORD:
Object record = model.newRecord(null, s);
Map<String,Object> rDefaults = defaults.get(s.getFullName());
for (Field f : s.getFields()) {
@@ -221,7 +221,7 @@ public class AvroColumnReader<D>
model.setField(record, f.name(), f.pos(), value);
}
return record;
- case ARRAY:
+ case ARRAY:
int length = values[column].nextLength();
List elements = (List)new GenericData.Array(length, s);
for (int i = 0; i < length; i++) {
@@ -254,7 +254,7 @@ public class AvroColumnReader<D>
private Object nextValue(Schema s, int column) throws IOException {
Object v = values[column].nextValue();
-
+
switch (s.getType()) {
case ENUM:
return model.createEnum(s.getEnumSymbols().get((Integer)v), s);
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnWriter.java
----------------------------------------------------------------------
diff --git a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnWriter.java b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnWriter.java
index abb5682..caf7fd5 100644
--- a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnWriter.java
+++ b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnWriter.java
@@ -89,14 +89,14 @@ public class AvroColumnWriter<D> {
assert(count == writer.getColumnCount());
writer.endRow();
}
-
+
private int write(Object o, Schema s, int column) throws IOException {
if (isSimple(s)) {
writeValue(o, s, column);
return column+1;
}
switch (s.getType()) {
- case MAP:
+ case MAP:
Map<?,?> map = (Map)o;
writer.writeLength(map.size(), column);
for (Map.Entry e : map.entrySet()) {
@@ -106,11 +106,11 @@ public class AvroColumnWriter<D> {
assert(c == column+arrayWidths[column]);
}
return column+arrayWidths[column];
- case RECORD:
+ case RECORD:
for (Field f : s.getFields())
column = write(model.getField(o,f.name(),f.pos()), f.schema(), column);
return column;
- case ARRAY:
+ case ARRAY:
Collection elements = (Collection)o;
writer.writeLength(elements.size(), column);
if (isSimple(s.getElementType())) { // optimize simple arrays
@@ -151,7 +151,7 @@ public class AvroColumnWriter<D> {
private void writeValue(Object value, Schema s, int column)
throws IOException {
-
+
switch (s.getType()) {
case STRING:
if (value instanceof Utf8) // convert Utf8 to String
@@ -160,7 +160,7 @@ public class AvroColumnWriter<D> {
case ENUM:
if (value instanceof Enum)
value = ((Enum)value).ordinal();
- else
+ else
value = s.getEnumOrdinal(value.toString());
break;
case FIXED:
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnator.java
----------------------------------------------------------------------
diff --git a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnator.java b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnator.java
index 2f9a3ef..a546c14 100644
--- a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnator.java
+++ b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnator.java
@@ -71,9 +71,9 @@ class AvroColumnator {
if (seen.containsKey(s)) // catch recursion
throw new TrevniRuntimeException("Cannot shred recursive schemas: "+s);
seen.put(s, s);
-
+
switch (s.getType()) {
- case MAP:
+ case MAP:
path = path == null ? ">" : path+">";
int start = columns.size();
ColumnMetaData p = addColumn(path, ValueType.NULL, parent, true);
@@ -85,7 +85,7 @@ class AvroColumnator {
for (Field field : s.getFields()) // flatten fields to columns
columnize(p(path, field.name(), "#"), field.schema(), parent, isArray);
break;
- case ARRAY:
+ case ARRAY:
path = path == null ? "[]" : path+"[]";
addArrayColumn(path, s.getElementType(), parent);
break;
@@ -131,7 +131,7 @@ class AvroColumnator {
// complex array: insert a parent column with lengths
int start = columns.size();
ColumnMetaData array = addColumn(path, ValueType.NULL, parent, true);
- columnize(path, element, array, false);
+ columnize(path, element, array, false);
arrayWidths.set(start, columns.size()-start); // fixup with actual width
}
@@ -139,8 +139,8 @@ class AvroColumnator {
switch (s.getType()) {
case NULL: case BOOLEAN:
case INT: case LONG:
- case FLOAT: case DOUBLE:
- case BYTES: case STRING:
+ case FLOAT: case DOUBLE:
+ case BYTES: case STRING:
case ENUM: case FIXED:
return true;
default:
@@ -165,4 +165,4 @@ class AvroColumnator {
}
}
-}
+}
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroTrevniInputFormat.java
----------------------------------------------------------------------
diff --git a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroTrevniInputFormat.java b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroTrevniInputFormat.java
index 47bec01..003c266 100644
--- a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroTrevniInputFormat.java
+++ b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroTrevniInputFormat.java
@@ -79,9 +79,9 @@ public class AvroTrevniInputFormat<T>
private long row;
public AvroWrapper<T> createKey() { return new AvroWrapper<T>(null); }
-
+
public NullWritable createValue() { return NullWritable.get(); }
-
+
public boolean next(AvroWrapper<T> wrapper, NullWritable ignore)
throws IOException {
if (!reader.hasNext())
@@ -90,13 +90,13 @@ public class AvroTrevniInputFormat<T>
row++;
return true;
}
-
+
public float getProgress() throws IOException { return row / rows; }
-
+
public long getPos() throws IOException { return row; }
public void close() throws IOException { reader.close(); }
-
+
};
}
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroTrevniOutputFormat.java
----------------------------------------------------------------------
diff --git a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroTrevniOutputFormat.java b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroTrevniOutputFormat.java
index 60b432b..ca71107 100644
--- a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroTrevniOutputFormat.java
+++ b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroTrevniOutputFormat.java
@@ -52,7 +52,7 @@ public class AvroTrevniOutputFormat <T>
/** The file name extension for trevni files. */
public final static String EXT = ".trv";
-
+
public static final String META_PREFIX = "trevni.meta.";
/** Add metadata to job output files.*/
@@ -84,7 +84,7 @@ public class AvroTrevniOutputFormat <T>
private AvroColumnWriter<T> writer =
new AvroColumnWriter<T>(schema, meta, ReflectData.get());
-
+
private void flush() throws IOException {
OutputStream out = fs.create(new Path(dir, "part-"+(part++)+EXT));
try {
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyInputFormat.java
----------------------------------------------------------------------
diff --git a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyInputFormat.java b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyInputFormat.java
index 89287e6..30d1682 100644
--- a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyInputFormat.java
+++ b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyInputFormat.java
@@ -29,28 +29,28 @@ import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
/**
* An {@link org.apache.hadoop.mapreduce.InputFormat} for Trevni files.
- *
+ *
* This implement was modeled off
* {@link org.apache.avro.mapreduce.AvroKeyInputFormat} to allow for easy
* transition
- *
+ *
* A MapReduce InputFormat that can handle Trevni container files.
*
* <p>Keys are AvroKey wrapper objects that contain the Trevni data. Since Trevni
* container files store only records (not key/value pairs), the value from
* this InputFormat is a NullWritable.</p>
- *
+ *
* <p>
* A subset schema to be read may be specified with
* {@link org.apache.avro.mapreduce.AvroJob#setInputKeySchema}.
*/
public class AvroTrevniKeyInputFormat<T> extends FileInputFormat<AvroKey<T>, NullWritable> {
-
+
@Override
public RecordReader<AvroKey<T>, NullWritable> createRecordReader(
InputSplit split, TaskAttemptContext context) throws IOException,
InterruptedException {
-
+
return new AvroTrevniKeyRecordReader<T>();
}
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyOutputFormat.java
----------------------------------------------------------------------
diff --git a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyOutputFormat.java b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyOutputFormat.java
index 34354f7..cc1f48f 100644
--- a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyOutputFormat.java
+++ b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyOutputFormat.java
@@ -29,17 +29,17 @@ import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
/** An {@link org.apache.hadoop.mapreduce.OutputFormat} that writes Avro data to
* Trevni files.
*
- * This implement was modeled off
+ * This implement was modeled off
* {@link org.apache.avro.mapreduce.AvroKeyOutputFormat} to allow for easy
* transition
- *
+ *
* FileOutputFormat for writing Trevni container files.
*
* <p>Since Trevni container files only contain records (not key/value pairs), this output
* format ignores the value.</p>
*
* @param <T> The (java) type of the Trevni data to write.
- *
+ *
* <p>Writes a directory of files per task, each comprising a single filesystem
* block. To reduce the number of files, increase the default filesystem block
* size for the job. Each task also requires enough memory to buffer a
@@ -50,7 +50,7 @@ public class AvroTrevniKeyOutputFormat <T> extends FileOutputFormat<AvroKey<T>,
@Override
public RecordWriter<AvroKey<T>, NullWritable> getRecordWriter(TaskAttemptContext context)
throws IOException, InterruptedException {
-
+
return new AvroTrevniKeyRecordWriter<T>(context );
}
}
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyRecordReader.java
----------------------------------------------------------------------
diff --git a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyRecordReader.java b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyRecordReader.java
index 88f2410..4751983 100644
--- a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyRecordReader.java
+++ b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyRecordReader.java
@@ -29,10 +29,10 @@ import org.apache.hadoop.io.NullWritable;
* @param <T> The (java) type of data in Trevni container file.
*/
public class AvroTrevniKeyRecordReader<T> extends AvroTrevniRecordReaderBase<AvroKey<T>, NullWritable, T> {
-
+
/** A reusable object to hold records of the Avro container file. */
private final AvroKey<T> mCurrentKey = new AvroKey<T>();
-
+
/** {@inheritDoc} */
@Override
public AvroKey<T> getCurrentKey() throws IOException,
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueInputFormat.java
----------------------------------------------------------------------
diff --git a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueInputFormat.java b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueInputFormat.java
index c16e381..2fb1b33 100644
--- a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueInputFormat.java
+++ b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueInputFormat.java
@@ -28,11 +28,11 @@ import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
/**
* An {@link org.apache.hadoop.mapreduce.InputFormat} for Trevni files.
- *
+ *
* This implement was modeled off
* {@link org.apache.avro.mapreduce.AvroKeyValueInputFormat} to allow for easy
* transition
- *
+ *
* <p>
* A MapReduce InputFormat that reads from Trevni container files of key/value generic records.
*
@@ -43,7 +43,7 @@ import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
*
* @param <K> The type of the Trevni key to read.
* @param <V> The type of the Trevni value to read.
- *
+ *
* <p>
* A subset schema to be read may be specified with
* {@link org.apache.avro.mapreduce.AvroJob#setInputKeySchema} and
@@ -56,7 +56,7 @@ public class AvroTrevniKeyValueInputFormat<K, V> extends FileInputFormat<AvroKe
public RecordReader<AvroKey<K>, AvroValue<V>> createRecordReader(
InputSplit split, TaskAttemptContext context) throws IOException,
InterruptedException {
-
+
return new AvroTrevniKeyValueRecordReader<K, V>();
}
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueOutputFormat.java
----------------------------------------------------------------------
diff --git a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueOutputFormat.java b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueOutputFormat.java
index c508df3..9f6b23f 100644
--- a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueOutputFormat.java
+++ b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueOutputFormat.java
@@ -28,11 +28,11 @@ import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
/** An {@link org.apache.hadoop.mapreduce.OutputFormat} that writes Avro data to
* Trevni files.
- *
- * This implement was modeled off
+ *
+ * This implement was modeled off
* {@link org.apache.avro.mapreduce.AvroKeyValueOutputFormat} to allow for easy
- * transition
- *
+ * transition
+ *
* * FileOutputFormat for writing Trevni container files of key/value pairs.
*
* <p>Since Trevni container files can only contain records (not key/value pairs), this
@@ -46,19 +46,19 @@ import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
*
* @param <K> The type of key. If an Avro type, it must be wrapped in an <code>AvroKey</code>.
* @param <V> The type of value. If an Avro type, it must be wrapped in an <code>AvroValue</code>.
- *
+ *
* <p>Writes a directory of files per task, each comprising a single filesystem
* block. To reduce the number of files, increase the default filesystem block
* size for the job. Each task also requires enough memory to buffer a
* filesystem block.
*/
-public class AvroTrevniKeyValueOutputFormat <K, V> extends FileOutputFormat<AvroKey<K>, AvroValue<V>> {
-
+public class AvroTrevniKeyValueOutputFormat <K, V> extends FileOutputFormat<AvroKey<K>, AvroValue<V>> {
+
/** {@inheritDoc} */
@Override
public RecordWriter<AvroKey<K>, AvroValue<V>> getRecordWriter(TaskAttemptContext context)
throws IOException, InterruptedException {
-
+
return new AvroTrevniKeyValueRecordWriter<K, V>(context );
}
}
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueRecordReader.java
----------------------------------------------------------------------
diff --git a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueRecordReader.java b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueRecordReader.java
index 3aa956f..31fe3f4 100644
--- a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueRecordReader.java
+++ b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueRecordReader.java
@@ -41,7 +41,7 @@ public class AvroTrevniKeyValueRecordReader<K, V> extends AvroTrevniRecordReader
private final AvroKey<K> mCurrentKey = new AvroKey<K>();
/** The current value the reader is on. */
private final AvroValue<V> mCurrentValue = new AvroValue<V>();
-
+
/** {@inheritDoc} */
@Override
public AvroKey<K> getCurrentKey() throws IOException,
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueRecordWriter.java
----------------------------------------------------------------------
diff --git a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueRecordWriter.java b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueRecordWriter.java
index 136ef06..9038028 100644
--- a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueRecordWriter.java
+++ b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniKeyValueRecordWriter.java
@@ -46,16 +46,16 @@ public class AvroTrevniKeyValueRecordWriter <K, V> extends AvroTrevniRecordWrite
/** The writer schema for the generic record entries of the Trevni container file. */
Schema mKeyValuePairSchema;
-
+
/** A reusable Avro generic record for writing key/value pairs to the file. */
AvroKeyValue<Object, Object> keyValueRecord;
-
+
/** A helper object that converts the input key to an Avro datum. */
AvroDatumConverter<K, ?> keyConverter;
-
+
/** A helper object that converts the input value to an Avro datum. */
AvroDatumConverter<V, ?> valueConverter;
-
+
/**
* Constructor.
* @param context The TaskAttempContext to supply the writer with information form the job configuration
@@ -63,30 +63,30 @@ public class AvroTrevniKeyValueRecordWriter <K, V> extends AvroTrevniRecordWrite
public AvroTrevniKeyValueRecordWriter(TaskAttemptContext context)
throws IOException {
super(context);
-
+
mKeyValuePairSchema = initSchema(context);
keyValueRecord = new AvroKeyValue<Object, Object>(new GenericData.Record(mKeyValuePairSchema));
}
-
+
/** {@inheritDoc} */
@Override
public void write(AvroKey<K> key, AvroValue<V> value) throws IOException,
InterruptedException {
-
+
keyValueRecord.setKey(key.datum());
keyValueRecord.setValue(value.datum());
writer.write(keyValueRecord.get());
if (writer.sizeEstimate() >= blockSize) // block full
flush();
}
-
+
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Override
protected Schema initSchema(TaskAttemptContext context) {
AvroDatumConverterFactory converterFactory = new AvroDatumConverterFactory(
context.getConfiguration());
-
+
keyConverter = converterFactory.create((Class<K>) context
.getOutputKeyClass());
valueConverter = converterFactory.create((Class<V>) context
@@ -95,7 +95,7 @@ public class AvroTrevniKeyValueRecordWriter <K, V> extends AvroTrevniRecordWrite
// Create the generic record schema for the key/value pair.
return AvroKeyValue.getSchema(
keyConverter.getWriterSchema(), valueConverter.getWriterSchema());
-
+
}
-
+
}
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniRecordReaderBase.java
----------------------------------------------------------------------
diff --git a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniRecordReaderBase.java b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniRecordReaderBase.java
index b68669f..1a3114f 100644
--- a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniRecordReaderBase.java
+++ b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniRecordReaderBase.java
@@ -37,16 +37,16 @@ import org.apache.trevni.avro.HadoopInput;
* @param <T> The type of the entries within the Trevni container file being read.
*/
public abstract class AvroTrevniRecordReaderBase<K, V, T> extends RecordReader<K, V> {
-
+
/** The Trevni file reader */
private AvroColumnReader<T> reader;
-
+
/** Number of rows in the Trevni file */
private float rows;
-
+
/** The current row number being read in */
private long row;
-
+
/** A reusable object to hold records of the Avro container file. */
private T mCurrentRecord;
@@ -60,11 +60,11 @@ public abstract class AvroTrevniRecordReaderBase<K, V, T> extends RecordReader<K
final AvroColumnReader.Params params =
new AvroColumnReader.Params(new HadoopInput(file.getPath(), context.getConfiguration()));
params.setModel(ReflectData.get());
-
+
if (AvroJob.getInputKeySchema(context.getConfiguration()) != null) {
params.setSchema(AvroJob.getInputKeySchema(context.getConfiguration()));
}
-
+
reader = new AvroColumnReader<T>(params);
rows = reader.getRowCount();
}
@@ -78,7 +78,7 @@ public abstract class AvroTrevniRecordReaderBase<K, V, T> extends RecordReader<K
row++;
return true;
}
-
+
/**
* Gets the current record read from the Trevni container file.
*
@@ -93,7 +93,7 @@ public abstract class AvroTrevniRecordReaderBase<K, V, T> extends RecordReader<K
/** {@inheritDoc} */
@Override
public void close() throws IOException {
- reader.close();
+ reader.close();
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniRecordWriterBase.java
----------------------------------------------------------------------
diff --git a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniRecordWriterBase.java b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniRecordWriterBase.java
index 94a332d..7f1d57b 100644
--- a/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniRecordWriterBase.java
+++ b/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/mapreduce/AvroTrevniRecordWriterBase.java
@@ -43,15 +43,15 @@ import org.apache.trevni.avro.AvroColumnWriter;
* @param <T> The type of the entries within the Trevni container file being writen.
*/
public abstract class AvroTrevniRecordWriterBase<K,V, T> extends RecordWriter<K, V> {
-
+
/** trevni file extension */
public final static String EXT = ".trv";
-
+
/** prefix of job configs that we care about */
public static final String META_PREFIX = "trevni.meta.";
-
- /** Counter that increments as new trevni files are create because the current file
- * has exceeded the block size
+
+ /** Counter that increments as new trevni files are create because the current file
+ * has exceeded the block size
* */
protected int part = 0;
@@ -60,31 +60,31 @@ public abstract class AvroTrevniRecordWriterBase<K,V, T> extends RecordWriter<K,
/** This will be a unique directory linked to the task */
final Path dirPath;
-
+
/** HDFS object */
final FileSystem fs;
/** Current configured blocksize */
final long blockSize;
-
+
/** Provided avro schema from the context */
protected Schema schema;
-
+
/** meta data to be stored in the output file. */
protected ColumnFileMetaData meta;
-
+
/**
* Constructor.
* @param context The TaskAttempContext to supply the writer with information form the job configuration
*/
public AvroTrevniRecordWriterBase(TaskAttemptContext context) throws IOException {
-
+
schema = initSchema(context);
meta = filterMetadata(context.getConfiguration());
writer = new AvroColumnWriter<T>(schema, meta, ReflectData.get());
Path outputPath = FileOutputFormat.getOutputPath(context);
-
+
String dir = FileOutputFormat.getUniqueFile(context, "part", "");
dirPath = new Path(outputPath.toString() + "/" + dir);
fs = dirPath.getFileSystem(context.getConfiguration());
@@ -97,8 +97,8 @@ public abstract class AvroTrevniRecordWriterBase<K,V, T> extends RecordWriter<K,
* Use the task context to construct a schema for writing
* @throws IOException
*/
- abstract protected Schema initSchema(TaskAttemptContext context);
-
+ abstract protected Schema initSchema(TaskAttemptContext context);
+
/**
* A Trevni flush will close the current file and prep a new writer
* @throws IOException
@@ -112,14 +112,14 @@ public abstract class AvroTrevniRecordWriterBase<K,V, T> extends RecordWriter<K,
}
writer = new AvroColumnWriter<T>(schema, meta, ReflectData.get());
}
-
+
/** {@inheritDoc} */
@Override
public void close(TaskAttemptContext arg0) throws IOException,
InterruptedException {
flush();
}
-
+
static ColumnFileMetaData filterMetadata(final Configuration configuration) {
final ColumnFileMetaData meta = new ColumnFileMetaData();
Iterator<Entry<String, String>> keyIterator = configuration.iterator();
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/RandomData.java
----------------------------------------------------------------------
diff --git a/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/RandomData.java b/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/RandomData.java
index 8f5976a..5d43aae 100644
--- a/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/RandomData.java
+++ b/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/RandomData.java
@@ -45,7 +45,7 @@ public class RandomData implements Iterable<Object> {
this.root = schema;
this.count = count;
}
-
+
public Iterator<Object> iterator() {
return new Iterator<Object>() {
private int n;
@@ -58,14 +58,14 @@ public class RandomData implements Iterable<Object> {
public void remove() { throw new UnsupportedOperationException(); }
};
}
-
+
@SuppressWarnings(value="unchecked")
private static Object generate(Schema schema, Random random, int d) {
switch (schema.getType()) {
case RECORD:
GenericRecord record = new GenericData.Record(schema);
for (Schema.Field field : schema.getFields()) {
- Object value = (field.getJsonProp(USE_DEFAULT) == null)
+ Object value = (field.getJsonProp(USE_DEFAULT) == null)
? generate(field.schema(), random, d+1)
: GenericData.get().getDefaultValue(field);
record.put(field.name(), value);
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestMetadataFiltering.java
----------------------------------------------------------------------
diff --git a/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestMetadataFiltering.java b/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestMetadataFiltering.java
index 920eb89..e3b9507 100644
--- a/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestMetadataFiltering.java
+++ b/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestMetadataFiltering.java
@@ -27,15 +27,15 @@ public class TestMetadataFiltering {
@Test public void testMetadataFiltering() throws Exception {
JobConf job = new JobConf();
-
+
job.set(AvroTrevniOutputFormat.META_PREFIX + "test1", "1");
job.set(AvroTrevniOutputFormat.META_PREFIX + "test2", "2");
job.set("test3", "3");
job.set(AvroJob.TEXT_PREFIX + "test4", "4");
job.set(AvroTrevniOutputFormat.META_PREFIX + "test5", "5");
-
+
ColumnFileMetaData metadata = AvroTrevniOutputFormat.filterMetadata(job);
-
+
assertTrue(metadata.get("test1") != null);
assertTrue(new String(metadata.get("test1")).equals("1"));
assertTrue(metadata.get("test2") != null);
@@ -44,7 +44,7 @@ public class TestMetadataFiltering {
assertTrue(new String(metadata.get("test5")).equals("5"));
assertTrue(metadata.get("test3") == null);
assertTrue(metadata.get("test4") == null);
-
+
}
-
+
}
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestShredder.java
----------------------------------------------------------------------
diff --git a/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestShredder.java b/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestShredder.java
index 06fdd09..9ca7344 100644
--- a/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestShredder.java
+++ b/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestShredder.java
@@ -78,7 +78,7 @@ public class TestShredder {
}
@Test public void testDefaultValue() throws Exception {
- String s =
+ String s =
"{\"type\":\"record\",\"name\":\"R\",\"fields\":["
+SIMPLE_FIELDS+","
+"{\"name\":\"z\",\"type\":\"int\","
@@ -89,7 +89,7 @@ public class TestShredder {
}
@Test public void testNestedRecord() throws Exception {
- String s =
+ String s =
"{\"type\":\"record\",\"name\":\"S\",\"fields\":["
+"{\"name\":\"x\",\"type\":\"int\"},"
+"{\"name\":\"R\",\"type\":"+SIMPLE_RECORD+"},"
@@ -103,25 +103,25 @@ public class TestShredder {
}
@Test public void testNamedRecord() throws Exception {
- String s =
- "{\"type\":\"record\",\"name\":\"S\",\"fields\":["
- +"{\"name\":\"R1\",\"type\":"+SIMPLE_RECORD+"},"
- +"{\"name\":\"R2\",\"type\":\"R\"}"
- +"]}";
- check(Schema.parse(s),
- new ColumnMetaData("R1#x", ValueType.INT),
- new ColumnMetaData("R1#y", ValueType.STRING),
- new ColumnMetaData("R2#x", ValueType.INT),
- new ColumnMetaData("R2#y", ValueType.STRING));
- }
-
+ String s =
+ "{\"type\":\"record\",\"name\":\"S\",\"fields\":["
+ +"{\"name\":\"R1\",\"type\":"+SIMPLE_RECORD+"},"
+ +"{\"name\":\"R2\",\"type\":\"R\"}"
+ +"]}";
+ check(Schema.parse(s),
+ new ColumnMetaData("R1#x", ValueType.INT),
+ new ColumnMetaData("R1#y", ValueType.STRING),
+ new ColumnMetaData("R2#x", ValueType.INT),
+ new ColumnMetaData("R2#y", ValueType.STRING));
+ }
+
@Test public void testSimpleArray() throws Exception {
String s = "{\"type\":\"array\",\"items\":\"long\"}";
check(Schema.parse(s),
new ColumnMetaData("[]", ValueType.LONG).isArray(true));
}
- private static final String RECORD_ARRAY =
+ private static final String RECORD_ARRAY =
"{\"type\":\"array\",\"items\":"+SIMPLE_RECORD+"}";
@Test public void testArray() throws Exception {
@@ -157,7 +157,7 @@ public class TestShredder {
}
@Test public void testNestedArray() throws Exception {
- String s =
+ String s =
"{\"type\":\"record\",\"name\":\"S\",\"fields\":["
+"{\"name\":\"x\",\"type\":\"int\"},"
+"{\"name\":\"A\",\"type\":"+RECORD_ARRAY+"},"
@@ -173,7 +173,7 @@ public class TestShredder {
}
@Test public void testNestedUnion() throws Exception {
- String s =
+ String s =
"{\"type\":\"record\",\"name\":\"S\",\"fields\":["
+"{\"name\":\"x\",\"type\":\"int\"},"
+"{\"name\":\"u\",\"type\":"+UNION+"},"
@@ -190,7 +190,7 @@ public class TestShredder {
}
@Test public void testUnionInArray() throws Exception {
- String s =
+ String s =
"{\"type\":\"record\",\"name\":\"S\",\"fields\":["
+"{\"name\":\"a\",\"type\":{\"type\":\"array\",\"items\":"+UNION+"}}"
+"]}";
@@ -209,7 +209,7 @@ public class TestShredder {
}
@Test public void testArrayInUnion() throws Exception {
- String s =
+ String s =
"{\"type\":\"record\",\"name\":\"S\",\"fields\":["
+"{\"name\":\"a\",\"type\":[\"int\","+RECORD_ARRAY+"]}]}";
ColumnMetaData q = new ColumnMetaData("a/array",ValueType.NULL)
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestWordCount.java
----------------------------------------------------------------------
diff --git a/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestWordCount.java b/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestWordCount.java
index d928a9f..79a8fff 100644
--- a/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestWordCount.java
+++ b/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestWordCount.java
@@ -62,7 +62,7 @@ public class TestWordCount {
collector.collect(new Pair<String,Long>(tokens.nextToken(),1L));
}
}
-
+
public static class ReduceImpl
extends AvroReducer<String, Long, Pair<String, Long> > {
@Override
@@ -74,7 +74,7 @@ public class TestWordCount {
sum += count;
collector.collect(new Pair<String,Long>(word, sum));
}
- }
+ }
@Test public void runTestsInOrder() throws Exception {
testOutputFormat();
@@ -87,26 +87,26 @@ public class TestWordCount {
public void testOutputFormat() throws Exception {
JobConf job = new JobConf();
-
+
WordCountUtil wordCountUtil = new WordCountUtil("trevniMapredTest");
-
+
wordCountUtil.writeLinesFile();
-
+
AvroJob.setInputSchema(job, STRING);
AvroJob.setOutputSchema(job, Pair.getPairSchema(STRING,LONG));
-
- AvroJob.setMapperClass(job, MapImpl.class);
+
+ AvroJob.setMapperClass(job, MapImpl.class);
AvroJob.setCombinerClass(job, ReduceImpl.class);
AvroJob.setReducerClass(job, ReduceImpl.class);
-
+
FileInputFormat.setInputPaths(job, new Path(wordCountUtil.getDir().toString() + "/in"));
FileOutputFormat.setOutputPath(job, new Path(wordCountUtil.getDir().toString() + "/out"));
FileOutputFormat.setCompressOutput(job, true);
-
+
job.setOutputFormat(AvroTrevniOutputFormat.class);
JobClient.runJob(job);
-
+
wordCountUtil.validateCountsFile();
}
@@ -118,20 +118,20 @@ public class TestWordCount {
total += (Long)r.get("value");
}
}
-
+
public void testInputFormat() throws Exception {
JobConf job = new JobConf();
WordCountUtil wordCountUtil = new WordCountUtil("trevniMapredTest");
-
-
+
+
Schema subSchema = Schema.parse("{\"type\":\"record\"," +
"\"name\":\"PairValue\","+
- "\"fields\": [ " +
- "{\"name\":\"value\", \"type\":\"long\"}" +
+ "\"fields\": [ " +
+ "{\"name\":\"value\", \"type\":\"long\"}" +
"]}");
AvroJob.setInputSchema(job, subSchema);
- AvroJob.setMapperClass(job, Counter.class);
+ AvroJob.setMapperClass(job, Counter.class);
FileInputFormat.setInputPaths(job, new Path(wordCountUtil.getDir().toString() + "/out/*"));
job.setInputFormat(AvroTrevniInputFormat.class);
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/WordCountUtil.java
----------------------------------------------------------------------
diff --git a/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/WordCountUtil.java b/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/WordCountUtil.java
index 68af7a3..775992a 100644
--- a/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/WordCountUtil.java
+++ b/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/WordCountUtil.java
@@ -60,13 +60,13 @@ public class WordCountUtil {
public WordCountUtil (String testName) {
this(testName, "part-00000");
}
-
+
public WordCountUtil (String testName, String partDirName) {
dir = new File("target/wc", testName);
linesFiles = new File(new File(dir, "in"), "lines.avro");
countFiles = new File(new File(dir, "out"), partDirName + "/part-0.trv");
}
-
+
public static final String[] LINES = new String[] {
"the quick brown fox jumps over the lazy dog",
"the cow jumps over the moon",
@@ -93,7 +93,7 @@ public class WordCountUtil {
public File getDir() {
return dir;
}
-
+
public void writeLinesFile() throws IOException {
FileUtil.fullyDelete(dir);
DatumWriter<String> writer = new GenericDatumWriter<String>();
@@ -117,7 +117,7 @@ public class WordCountUtil {
reader.close();
assertEquals(COUNTS.size(), numWords);
}
-
+
public void validateCountsFileGenericRecord() throws Exception {
AvroColumnReader<GenericRecord > reader =
new AvroColumnReader<GenericRecord >
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/mapreduce/TestKeyValueWordCount.java
----------------------------------------------------------------------
diff --git a/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/mapreduce/TestKeyValueWordCount.java b/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/mapreduce/TestKeyValueWordCount.java
index deea1ca..47cbee4 100644
--- a/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/mapreduce/TestKeyValueWordCount.java
+++ b/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/mapreduce/TestKeyValueWordCount.java
@@ -43,13 +43,13 @@ import org.apache.trevni.avro.WordCountUtil;
import org.junit.Test;
public class TestKeyValueWordCount {
-
+
private static long total = 0;
static final Schema STRING = Schema.create(Schema.Type.STRING);
static { GenericData.setStringType(STRING, GenericData.StringType.String); }
static final Schema LONG = Schema.create(Schema.Type.LONG);
-
+
private static class WordCountMapper extends
Mapper<AvroKey<String>, NullWritable, Text, LongWritable> {
private LongWritable mCount = new LongWritable();
@@ -76,12 +76,12 @@ public class TestKeyValueWordCount {
}
}
-
+
private static class WordCountReducer extends Reducer< Text, LongWritable, AvroKey<String>, AvroValue<Long>> {
-
+
AvroKey<String> resultKey = new AvroKey<String>();
AvroValue<Long> resultValue = new AvroValue<Long>();
-
+
@Override
protected void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException {
long sum = 0;
@@ -90,11 +90,11 @@ public class TestKeyValueWordCount {
}
resultKey.datum(key.toString());
resultValue.datum(sum);
-
+
context.write(resultKey, resultValue);
}
}
-
+
public static class Counter extends
Mapper<AvroKey<String>, AvroValue<Long>, NullWritable, NullWritable> {
@Override
@@ -102,8 +102,8 @@ public class TestKeyValueWordCount {
throws IOException, InterruptedException {
total += value.datum();
}
- }
-
+ }
+
@Test public void testIOFormat() throws Exception {
checkOutputFormat();
checkInputFormat();
@@ -111,49 +111,49 @@ public class TestKeyValueWordCount {
public void checkOutputFormat() throws Exception {
Job job = new Job();
-
+
WordCountUtil wordCountUtil = new WordCountUtil("trevniMapReduceKeyValueTest", "part-r-00000");
-
+
wordCountUtil.writeLinesFile();
-
+
AvroJob.setInputKeySchema(job, STRING);
AvroJob.setOutputKeySchema(job, STRING);
AvroJob.setOutputValueSchema(job, LONG);
-
+
job.setMapperClass(WordCountMapper.class);
job.setReducerClass(WordCountReducer.class);
-
+
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(LongWritable.class);
-
+
FileInputFormat.setInputPaths(job, new Path(wordCountUtil.getDir().toString() + "/in"));
FileOutputFormat.setOutputPath(job, new Path(wordCountUtil.getDir().toString() + "/out"));
FileOutputFormat.setCompressOutput(job, true);
-
+
job.setInputFormatClass(AvroKeyInputFormat.class);
job.setOutputFormatClass(AvroTrevniKeyValueOutputFormat.class);
job.waitForCompletion(true);
-
+
wordCountUtil.validateCountsFileGenericRecord();
}
-
+
public void checkInputFormat() throws Exception {
Job job = new Job();
-
+
WordCountUtil wordCountUtil = new WordCountUtil("trevniMapReduceKeyValueTest");
-
+
job.setMapperClass(Counter.class);
-
+
FileInputFormat.setInputPaths(job, new Path(wordCountUtil.getDir().toString() + "/out/*"));
job.setInputFormatClass(AvroTrevniKeyValueInputFormat.class);
-
+
job.setNumReduceTasks(0);
job.setOutputFormatClass(NullOutputFormat.class);
-
+
total = 0;
job.waitForCompletion(true);
assertEquals(WordCountUtil.TOTAL, total);
-
+
}
}
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/mapreduce/TestKeyWordCount.java
----------------------------------------------------------------------
diff --git a/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/mapreduce/TestKeyWordCount.java b/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/mapreduce/TestKeyWordCount.java
index 8623fb4..1ba50ff 100644
--- a/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/mapreduce/TestKeyWordCount.java
+++ b/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/mapreduce/TestKeyWordCount.java
@@ -45,14 +45,14 @@ import org.apache.trevni.avro.mapreduce.AvroTrevniKeyOutputFormat;
import org.junit.Test;
public class TestKeyWordCount {
-
+
private static long total = 0;
static final Schema STRING = Schema.create(Schema.Type.STRING);
static { GenericData.setStringType(STRING, GenericData.StringType.String); }
static final Schema LONG = Schema.create(Schema.Type.LONG);
-
-
+
+
private static class WordCountMapper extends
Mapper<AvroKey<String>, NullWritable, Text, LongWritable> {
private LongWritable mCount = new LongWritable();
@@ -79,33 +79,33 @@ public class TestKeyWordCount {
}
}
-
+
private static class WordCountReducer extends Reducer< Text, LongWritable, AvroKey<GenericData.Record>, NullWritable> {
-
+
private AvroKey<GenericData.Record> result ;
-
+
@Override
protected void setup(Context context) {
result = new AvroKey<GenericData.Record>();
result.datum(new Record(Pair.getPairSchema(STRING,LONG)));
}
-
+
@Override
protected void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException {
long count = 0;
for (LongWritable value: values) {
count += value.get();
}
-
+
result.datum().put("key", key.toString());
result.datum().put("value", count);
-
+
context.write(result, NullWritable.get());
}
}
-
-
+
+
public static class Counter extends
Mapper<AvroKey<GenericData.Record>, NullWritable, NullWritable, NullWritable> {
@Override
@@ -114,8 +114,8 @@ public class TestKeyWordCount {
total += (Long)key.datum().get("value");
}
}
-
-
+
+
@Test public void testIOFormat() throws Exception {
checkOutputFormat();
checkInputFormat();
@@ -123,56 +123,56 @@ public class TestKeyWordCount {
public void checkOutputFormat() throws Exception {
Job job = new Job();
-
+
WordCountUtil wordCountUtil = new WordCountUtil("trevniMapReduceKeyTest", "part-r-00000");
-
+
wordCountUtil.writeLinesFile();
-
+
AvroJob.setInputKeySchema(job, STRING);
AvroJob.setOutputKeySchema(job, Pair.getPairSchema(STRING,LONG));
-
+
job.setMapperClass(WordCountMapper.class);
job.setReducerClass(WordCountReducer.class);
-
+
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(LongWritable.class);
-
+
FileInputFormat.setInputPaths(job, new Path(wordCountUtil.getDir().toString() + "/in"));
FileOutputFormat.setOutputPath(job, new Path(wordCountUtil.getDir().toString() + "/out"));
FileOutputFormat.setCompressOutput(job, true);
-
+
job.setInputFormatClass(AvroKeyInputFormat.class);
job.setOutputFormatClass(AvroTrevniKeyOutputFormat.class);
job.waitForCompletion(true);
-
+
wordCountUtil.validateCountsFile();
}
-
+
public void checkInputFormat() throws Exception {
Job job = new Job();
-
+
WordCountUtil wordCountUtil = new WordCountUtil("trevniMapReduceKeyTest");
-
+
job.setMapperClass(Counter.class);
Schema subSchema = Schema.parse("{\"type\":\"record\"," +
"\"name\":\"PairValue\","+
- "\"fields\": [ " +
- "{\"name\":\"value\", \"type\":\"long\"}" +
+ "\"fields\": [ " +
+ "{\"name\":\"value\", \"type\":\"long\"}" +
"]}");
AvroJob.setInputKeySchema(job, subSchema);
-
+
FileInputFormat.setInputPaths(job, new Path(wordCountUtil.getDir().toString() + "/out/*"));
job.setInputFormatClass(AvroTrevniKeyInputFormat.class);
-
+
job.setNumReduceTasks(0);
job.setOutputFormatClass(NullOutputFormat.class);
-
+
total = 0;
job.waitForCompletion(true);
assertEquals(WordCountUtil.TOTAL, total);
-
+
}
-
+
}
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/core/pom.xml
----------------------------------------------------------------------
diff --git a/lang/java/trevni/core/pom.xml b/lang/java/trevni/core/pom.xml
index 8e96aa9..084b167 100644
--- a/lang/java/trevni/core/pom.xml
+++ b/lang/java/trevni/core/pom.xml
@@ -37,7 +37,7 @@
<artifactId>snappy-java</artifactId>
<version>${snappy.version}</version>
<scope>compile</scope>
- </dependency>
+ </dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/core/src/main/java/org/apache/trevni/BZip2Codec.java
----------------------------------------------------------------------
diff --git a/lang/java/trevni/core/src/main/java/org/apache/trevni/BZip2Codec.java b/lang/java/trevni/core/src/main/java/org/apache/trevni/BZip2Codec.java
index 1c0d64b..9513ed4 100644
--- a/lang/java/trevni/core/src/main/java/org/apache/trevni/BZip2Codec.java
+++ b/lang/java/trevni/core/src/main/java/org/apache/trevni/BZip2Codec.java
@@ -29,7 +29,7 @@ public class BZip2Codec extends Codec {
private ByteArrayOutputStream outputBuffer;
public static final int DEFAULT_BUFFER_SIZE = 64 * 1024;
-
+
@Override
ByteBuffer compress(ByteBuffer uncompressedData) throws IOException {
ByteArrayOutputStream baos = getOutputBuffer(uncompressedData.remaining());
@@ -55,23 +55,23 @@ public class BZip2Codec extends Codec {
byte[] buffer = new byte[DEFAULT_BUFFER_SIZE];
int readCount = -1;
-
+
while ( (readCount = inputStream.read(buffer, compressedData.position(), buffer.length))> 0) {
baos.write(buffer, 0, readCount);
}
-
+
ByteBuffer result = ByteBuffer.wrap(baos.toByteArray());
return result;
} finally {
inputStream.close();
}
}
-
+
private ByteArrayOutputStream getOutputBuffer(int suggestedLength) {
if (null == outputBuffer)
outputBuffer = new ByteArrayOutputStream(suggestedLength);
outputBuffer.reset();
return outputBuffer;
}
-
+
}
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/core/src/main/java/org/apache/trevni/BlockDescriptor.java
----------------------------------------------------------------------
diff --git a/lang/java/trevni/core/src/main/java/org/apache/trevni/BlockDescriptor.java b/lang/java/trevni/core/src/main/java/org/apache/trevni/BlockDescriptor.java
index 5f2c0f3..cbb15bd 100644
--- a/lang/java/trevni/core/src/main/java/org/apache/trevni/BlockDescriptor.java
+++ b/lang/java/trevni/core/src/main/java/org/apache/trevni/BlockDescriptor.java
@@ -25,13 +25,13 @@ class BlockDescriptor {
int compressedSize;
BlockDescriptor() {}
-
+
BlockDescriptor(int rowCount, int uncompressedSize, int compressedSize) {
this.rowCount = rowCount;
this.uncompressedSize = uncompressedSize;
this.compressedSize = compressedSize;
}
-
+
public void writeTo(OutputBuffer out) throws IOException {
out.writeFixed32(rowCount);
out.writeFixed32(uncompressedSize);
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnDescriptor.java
----------------------------------------------------------------------
diff --git a/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnDescriptor.java b/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnDescriptor.java
index 6a052d1..3cbabac 100644
--- a/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnDescriptor.java
+++ b/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnDescriptor.java
@@ -76,7 +76,7 @@ class ColumnDescriptor<T extends Comparable> {
firstValues[i] = in.<T>readValue(metaData.getType());
}
dataStart = in.tell();
-
+
// compute blockStarts and firstRows
Checksum checksum = Checksum.get(metaData);
blockStarts = new long[blocks.length];
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnFileReader.java
----------------------------------------------------------------------
diff --git a/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnFileReader.java b/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnFileReader.java
index ec080b8..cb475f5 100644
--- a/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnFileReader.java
+++ b/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnFileReader.java
@@ -131,7 +131,7 @@ public class ColumnFileReader implements Closeable {
for (int i = 0; i < columnCount; i++)
columns[i].start = in.readFixed64();
}
-
+
/** Return an iterator over values in the named column. */
public <T extends Comparable> ColumnValues<T> getValues(String columnName)
throws IOException {
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnFileWriter.java
----------------------------------------------------------------------
diff --git a/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnFileWriter.java b/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnFileWriter.java
index 0f4a21e..5eb92aa 100644
--- a/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnFileWriter.java
+++ b/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnFileWriter.java
@@ -68,7 +68,7 @@ public class ColumnFileWriter {
if (parent != null && !seen.contains(parent.getName()))
throw new TrevniRuntimeException("Parent must precede child: "+name);
seen.add(name);
- }
+ }
}
void incrementSize(int n) { size += n; }
@@ -133,7 +133,7 @@ public class ColumnFileWriter {
/** Write all rows added to the named output stream. */
public void writeTo(OutputStream out) throws IOException {
writeHeader(out);
-
+
for (int column = 0; column < columnCount; column++)
columns[column].writeTo(out);
}
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnOutputBuffer.java
----------------------------------------------------------------------
diff --git a/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnOutputBuffer.java b/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnOutputBuffer.java
index b689915..3217753 100644
--- a/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnOutputBuffer.java
+++ b/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnOutputBuffer.java
@@ -90,9 +90,9 @@ class ColumnOutputBuffer {
? firstValues.get(firstValues.size()-1).length
: 0)
+ data.position(); // data
-
+
writer.incrementSize(sizeIncrement);
- size += sizeIncrement;
+ size += sizeIncrement;
buffer = new OutputBuffer();
rowCount = 0;
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnValues.java
----------------------------------------------------------------------
diff --git a/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnValues.java b/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnValues.java
index a47fc85..9df153f 100644
--- a/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnValues.java
+++ b/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnValues.java
@@ -34,7 +34,7 @@ public class ColumnValues<T extends Comparable>
private InputBuffer values;
private int block = -1;
private long row = 0;
- private T previous;
+ private T previous;
private int arrayLength;
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/core/src/main/java/org/apache/trevni/DeflateCodec.java
----------------------------------------------------------------------
diff --git a/lang/java/trevni/core/src/main/java/org/apache/trevni/DeflateCodec.java b/lang/java/trevni/core/src/main/java/org/apache/trevni/DeflateCodec.java
index 7e9a0be..96077bb 100644
--- a/lang/java/trevni/core/src/main/java/org/apache/trevni/DeflateCodec.java
+++ b/lang/java/trevni/core/src/main/java/org/apache/trevni/DeflateCodec.java
@@ -45,13 +45,13 @@ class DeflateCodec extends Codec {
writeAndClose(data, new InflaterOutputStream(baos, getInflater()));
return ByteBuffer.wrap(baos.toByteArray());
}
-
+
private void writeAndClose(ByteBuffer data, OutputStream out)
throws IOException {
out.write(data.array(), data.position(), data.remaining());
out.close();
}
-
+
private Inflater getInflater() {
if (null == inflater)
inflater = new Inflater(true);
@@ -65,7 +65,7 @@ class DeflateCodec extends Codec {
deflater.reset();
return deflater;
}
-
+
private ByteArrayOutputStream getOutputBuffer(int suggestedLength) {
if (null == outputBuffer)
outputBuffer = new ByteArrayOutputStream(suggestedLength);
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/core/src/main/java/org/apache/trevni/InputBuffer.java
----------------------------------------------------------------------
diff --git a/lang/java/trevni/core/src/main/java/org/apache/trevni/InputBuffer.java b/lang/java/trevni/core/src/main/java/org/apache/trevni/InputBuffer.java
index e3bd415..4023731 100644
--- a/lang/java/trevni/core/src/main/java/org/apache/trevni/InputBuffer.java
+++ b/lang/java/trevni/core/src/main/java/org/apache/trevni/InputBuffer.java
@@ -35,7 +35,7 @@ class InputBuffer {
private int limit; // end of valid buffer data
private CharsetDecoder utf8 = Charset.forName("UTF-8").newDecoder();
-
+
private int bitCount; // position in booleans
private int runLength; // length of run
@@ -233,7 +233,7 @@ class InputBuffer {
}
return (l >>> 1) ^ -(l & 1); // back to two's-complement
}
-
+
// splitting readLong up makes it faster because of the JVM does more
// optimizations on small methods
private long innerLongDecode(long l) throws IOException {
@@ -302,7 +302,7 @@ class InputBuffer {
byte[] bytes = new byte[length];
readFully(bytes, 0, length);
return utf8.decode(ByteBuffer.wrap(bytes, 0, length)).toString();
- }
+ }
public byte[] readBytes() throws IOException {
byte[] result = new byte[readInt()];
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/core/src/main/java/org/apache/trevni/MetaData.java
----------------------------------------------------------------------
diff --git a/lang/java/trevni/core/src/main/java/org/apache/trevni/MetaData.java b/lang/java/trevni/core/src/main/java/org/apache/trevni/MetaData.java
index 648d8a1..08d0072 100644
--- a/lang/java/trevni/core/src/main/java/org/apache/trevni/MetaData.java
+++ b/lang/java/trevni/core/src/main/java/org/apache/trevni/MetaData.java
@@ -44,7 +44,7 @@ public class MetaData<T extends MetaData> extends LinkedHashMap<String,byte[]> {
setReserved(CODEC_KEY, codec);
return (T)this;
}
-
+
/** Return the checksum algorithm name. */
public String getChecksum() { return getString(CHECKSUM_KEY); }
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/core/src/main/java/org/apache/trevni/OutputBuffer.java
----------------------------------------------------------------------
diff --git a/lang/java/trevni/core/src/main/java/org/apache/trevni/OutputBuffer.java b/lang/java/trevni/core/src/main/java/org/apache/trevni/OutputBuffer.java
index 41174fb..0bba8ac 100644
--- a/lang/java/trevni/core/src/main/java/org/apache/trevni/OutputBuffer.java
+++ b/lang/java/trevni/core/src/main/java/org/apache/trevni/OutputBuffer.java
@@ -98,7 +98,7 @@ class OutputBuffer extends ByteArrayOutputStream {
int len = bytes.limit() - pos;
writeBytes(bytes.array(), start, len);
}
-
+
public void writeBytes(byte[] bytes) throws IOException {
writeBytes(bytes, 0, bytes.length);
}
@@ -158,7 +158,7 @@ class OutputBuffer extends ByteArrayOutputStream {
}
}
}
- }
+ }
buf[count++] = (byte) n;
}
@@ -203,7 +203,7 @@ class OutputBuffer extends ByteArrayOutputStream {
}
buf[count++] = (byte) n;
}
-
+
private void ensure(int n) {
if (count + n > buf.length)
buf = Arrays.copyOf(buf, Math.max(buf.length << 1, count + n));
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/core/src/main/java/org/apache/trevni/ValueType.java
----------------------------------------------------------------------
diff --git a/lang/java/trevni/core/src/main/java/org/apache/trevni/ValueType.java b/lang/java/trevni/core/src/main/java/org/apache/trevni/ValueType.java
index 712a7d9..dddcb5a 100644
--- a/lang/java/trevni/core/src/main/java/org/apache/trevni/ValueType.java
+++ b/lang/java/trevni/core/src/main/java/org/apache/trevni/ValueType.java
@@ -30,5 +30,5 @@ public enum ValueType {
public static ValueType forName(String name) {
return valueOf(name.toUpperCase());
}
-
+
}
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/core/src/test/java/org/apache/trevni/TestBZip2Codec.java
----------------------------------------------------------------------
diff --git a/lang/java/trevni/core/src/test/java/org/apache/trevni/TestBZip2Codec.java b/lang/java/trevni/core/src/test/java/org/apache/trevni/TestBZip2Codec.java
index c4a39de..ac7c81b 100644
--- a/lang/java/trevni/core/src/test/java/org/apache/trevni/TestBZip2Codec.java
+++ b/lang/java/trevni/core/src/test/java/org/apache/trevni/TestBZip2Codec.java
@@ -25,42 +25,42 @@ import java.nio.ByteBuffer;
import org.junit.Test;
public class TestBZip2Codec {
-
+
@Test
public void testBZip2CompressionAndDecompression() throws IOException {
-
+
MetaData meta = new MetaData();
meta.setCodec("bzip2");
Codec codec = Codec.get(meta);
-
+
//Confirm that the right codec Came back
assertTrue(codec instanceof BZip2Codec);
-
+
//This is 3 times the byte buffer on the BZip2 decompress plus some extra
final int inputByteSize = BZip2Codec.DEFAULT_BUFFER_SIZE * 3 + 42;
-
+
byte[] inputByteArray = new byte[inputByteSize];
-
+
//Generate something that will compress well
for (int i = 0; i < inputByteSize; i++) {
inputByteArray[i] = (byte)(65 + i % 10);
}
-
+
ByteBuffer inputByteBuffer = ByteBuffer.wrap(inputByteArray);
-
+
ByteBuffer compressedBuffer = codec.compress(inputByteBuffer);
-
+
//Make sure something returned
assertTrue(compressedBuffer.array().length > 0);
//Make sure the compressed output is smaller then the original
assertTrue(compressedBuffer.array().length < inputByteArray.length);
-
+
ByteBuffer decompressedBuffer = codec.decompress(compressedBuffer);
-
+
//The original array should be the same length as the decompressed array
assertTrue(decompressedBuffer.array().length == inputByteArray.length);
-
- //Every byte in the outputByteArray should equal every byte in the input array
+
+ //Every byte in the outputByteArray should equal every byte in the input array
byte[] outputByteArray = decompressedBuffer.array();
for (int i = 0; i < inputByteSize; i++) {
inputByteArray[i] = outputByteArray[i];
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/core/src/test/java/org/apache/trevni/TestIOBuffers.java
----------------------------------------------------------------------
diff --git a/lang/java/trevni/core/src/test/java/org/apache/trevni/TestIOBuffers.java b/lang/java/trevni/core/src/test/java/org/apache/trevni/TestIOBuffers.java
index 707848e..a0b1068 100644
--- a/lang/java/trevni/core/src/test/java/org/apache/trevni/TestIOBuffers.java
+++ b/lang/java/trevni/core/src/test/java/org/apache/trevni/TestIOBuffers.java
@@ -52,7 +52,7 @@ public class TestIOBuffers {
OutputBuffer out = new OutputBuffer();
for (int i = 0; i < COUNT; i++)
out.writeValue(random.nextBoolean(), ValueType.BOOLEAN);
-
+
InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray()));
random = TestUtil.createRandom();
for (int i = 0; i < COUNT; i++)
@@ -65,7 +65,7 @@ public class TestIOBuffers {
OutputBuffer out = new OutputBuffer();
for (int i = 0; i < COUNT; i++)
out.writeInt(random.nextInt());
-
+
InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray()));
random = TestUtil.createRandom();
for (int i = 0; i < COUNT; i++)
@@ -77,7 +77,7 @@ public class TestIOBuffers {
OutputBuffer out = new OutputBuffer();
for (int i = 0; i < COUNT; i++)
out.writeLong(random.nextLong());
-
+
InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray()));
random = TestUtil.createRandom();
for (int i = 0; i < COUNT; i++)
@@ -89,7 +89,7 @@ public class TestIOBuffers {
OutputBuffer out = new OutputBuffer();
for (int i = 0; i < COUNT; i++)
out.writeFixed32(random.nextInt());
-
+
InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray()));
random = TestUtil.createRandom();
for (int i = 0; i < COUNT; i++)
@@ -101,41 +101,41 @@ public class TestIOBuffers {
OutputBuffer out = new OutputBuffer();
for (int i = 0; i < COUNT; i++)
out.writeFixed64(random.nextLong());
-
+
InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray()));
random = TestUtil.createRandom();
for (int i = 0; i < COUNT; i++)
Assert.assertEquals(random.nextLong(), in.readFixed64());
}
-
+
@Test public void testFloat() throws Exception {
Random random = TestUtil.createRandom();
OutputBuffer out = new OutputBuffer();
for (int i = 0; i < COUNT; i++)
out.writeFloat(random.nextFloat());
-
+
InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray()));
random = TestUtil.createRandom();
for (int i = 0; i < COUNT; i++)
Assert.assertEquals(random.nextFloat(), in.readFloat(), 0);
}
-
+
@Test public void testDouble() throws Exception {
OutputBuffer out = new OutputBuffer();
for (int i = 0; i < COUNT; i++)
out.writeDouble(Double.MIN_VALUE);
-
+
InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray()));
for (int i = 0; i < COUNT; i++)
Assert.assertEquals(Double.MIN_VALUE, in.readDouble(), 0);
}
-
+
@Test public void testBytes() throws Exception {
Random random = TestUtil.createRandom();
OutputBuffer out = new OutputBuffer();
for (int i = 0; i < COUNT; i++)
out.writeBytes(TestUtil.randomBytes(random));
-
+
InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray()));
random = TestUtil.createRandom();
for (int i = 0; i < COUNT; i++)
@@ -147,7 +147,7 @@ public class TestIOBuffers {
OutputBuffer out = new OutputBuffer();
for (int i = 0; i < COUNT; i++)
out.writeString(TestUtil.randomString(random));
-
+
InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray()));
random = TestUtil.createRandom();
for (int i = 0; i < COUNT; i++)
@@ -158,7 +158,7 @@ public class TestIOBuffers {
OutputBuffer out = new OutputBuffer();
out.writeValue(null, ValueType.NULL);
out.writeLong(sentinel);
-
+
InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray()));
in.skipValue(ValueType.NULL);
Assert.assertEquals(sentinel, in.readLong());
@@ -168,7 +168,7 @@ public class TestIOBuffers {
OutputBuffer out = new OutputBuffer();
out.writeValue(false, ValueType.BOOLEAN);
out.writeLong(sentinel);
-
+
InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray()));
in.skipValue(ValueType.BOOLEAN);
Assert.assertEquals(sentinel, in.readLong());
@@ -178,7 +178,7 @@ public class TestIOBuffers {
OutputBuffer out = new OutputBuffer();
out.writeValue(Integer.MAX_VALUE, ValueType.INT);
out.writeLong(sentinel);
-
+
InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray()));
in.skipValue(ValueType.INT);
Assert.assertEquals(sentinel, in.readLong());
@@ -188,7 +188,7 @@ public class TestIOBuffers {
OutputBuffer out = new OutputBuffer();
out.writeValue(Long.MAX_VALUE, ValueType.LONG);
out.writeLong(sentinel);
-
+
InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray()));
in.skipValue(ValueType.LONG);
Assert.assertEquals(sentinel, in.readLong());
@@ -198,7 +198,7 @@ public class TestIOBuffers {
OutputBuffer out = new OutputBuffer();
out.writeValue(Integer.MAX_VALUE, ValueType.FIXED32);
out.writeLong(sentinel);
-
+
InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray()));
in.skipValue(ValueType.LONG);
Assert.assertEquals(sentinel, in.readLong());
@@ -208,7 +208,7 @@ public class TestIOBuffers {
OutputBuffer out = new OutputBuffer();
out.writeValue(Long.MAX_VALUE, ValueType.FIXED64);
out.writeLong(sentinel);
-
+
InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray()));
in.skipValue(ValueType.LONG);
Assert.assertEquals(sentinel, in.readLong());
@@ -218,7 +218,7 @@ public class TestIOBuffers {
OutputBuffer out = new OutputBuffer();
out.writeValue(Float.MAX_VALUE, ValueType.FLOAT);
out.writeLong(sentinel);
-
+
InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray()));
in.skipValue(ValueType.FLOAT);
Assert.assertEquals(sentinel, in.readLong());
@@ -228,7 +228,7 @@ public class TestIOBuffers {
OutputBuffer out = new OutputBuffer();
out.writeValue(Double.MAX_VALUE, ValueType.DOUBLE);
out.writeLong(sentinel);
-
+
InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray()));
in.skipValue(ValueType.DOUBLE);
Assert.assertEquals(sentinel, in.readLong());
@@ -238,7 +238,7 @@ public class TestIOBuffers {
OutputBuffer out = new OutputBuffer();
out.writeValue("trevni", ValueType.STRING);
out.writeLong(sentinel);
-
+
InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray()));
in.skipValue(ValueType.STRING);
Assert.assertEquals(sentinel, in.readLong());
@@ -248,7 +248,7 @@ public class TestIOBuffers {
OutputBuffer out = new OutputBuffer();
out.writeValue("trevni".getBytes(), ValueType.BYTES);
out.writeLong(sentinel);
-
+
InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray()));
in.skipValue(ValueType.BYTES);
Assert.assertEquals(sentinel, in.readLong());
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/core/src/test/java/org/apache/trevni/TestInputBytes.java
----------------------------------------------------------------------
diff --git a/lang/java/trevni/core/src/test/java/org/apache/trevni/TestInputBytes.java b/lang/java/trevni/core/src/test/java/org/apache/trevni/TestInputBytes.java
index c4cb2aa..c55b532 100644
--- a/lang/java/trevni/core/src/test/java/org/apache/trevni/TestInputBytes.java
+++ b/lang/java/trevni/core/src/test/java/org/apache/trevni/TestInputBytes.java
@@ -38,7 +38,7 @@ public class TestInputBytes {
random.nextBytes(data);
Input in = new InputBytes(data);
-
+
for (int i = 0; i < COUNT; i++) {
int p = random.nextInt(length);
int l = Math.min(random.nextInt(SIZE/10), length-p);
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/java/trevni/core/src/test/java/org/apache/trevni/TestUtil.java
----------------------------------------------------------------------
diff --git a/lang/java/trevni/core/src/test/java/org/apache/trevni/TestUtil.java b/lang/java/trevni/core/src/test/java/org/apache/trevni/TestUtil.java
index ab4796d..22bbae9 100644
--- a/lang/java/trevni/core/src/test/java/org/apache/trevni/TestUtil.java
+++ b/lang/java/trevni/core/src/test/java/org/apache/trevni/TestUtil.java
@@ -39,7 +39,7 @@ public class TestUtil {
String configured = System.getProperty("test.seed");
if (configured != null)
seed = Long.valueOf(configured);
- else
+ else
seed = System.currentTimeMillis();
System.err.println("test.seed="+seed);
seedSet = true;
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/py/build.xml
----------------------------------------------------------------------
diff --git a/lang/py/build.xml b/lang/py/build.xml
index 61c3f4c..5ef5214 100644
--- a/lang/py/build.xml
+++ b/lang/py/build.xml
@@ -17,7 +17,7 @@
-->
<project name="Avro" default="dist" xmlns:ivy="antlib:org.apache.ivy.ant">
-
+
<!-- Load user's default properties. -->
<property file="${user.home}/build.properties"/>
@@ -66,7 +66,7 @@
</classpath>
</typedef>
</target>
-
+
<target name="ivy-download" unless="ivy.jar.found" >
<get src="http://repo2.maven.org/maven2/org/apache/ivy/ivy/${ivy.version}/ivy-${ivy.version}.jar" dest="${ivy.jar}" usetimestamp="true" />
</target>
@@ -79,13 +79,13 @@
<fileset dir="${src.dir}">
<exclude name="**/*.pyc"/>
<exclude name="**/*.py~"/>
- </fileset>
+ </fileset>
</copy>
<copy todir="${build.dir}/test">
<fileset dir="${test.dir}">
<exclude name="**/*.pyc"/>
<exclude name="**/*.py~"/>
- </fileset>
+ </fileset>
</copy>
<copy todir="${build.dir}/lib">
<fileset dir="${lib.dir}" />
@@ -102,9 +102,9 @@
toFile="${build.dir}/src/avro/ipc.py"
overwrite="true">
<filterset>
- <filter token="HANDSHAKE_REQUEST_SCHEMA"
+ <filter token="HANDSHAKE_REQUEST_SCHEMA"
value="${handshake.request.json}"/>
- <filter token="HANDSHAKE_RESPONSE_SCHEMA"
+ <filter token="HANDSHAKE_RESPONSE_SCHEMA"
value="${handshake.response.json}"/>
</filterset>
</copy>
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/py/ivysettings.xml
----------------------------------------------------------------------
diff --git a/lang/py/ivysettings.xml b/lang/py/ivysettings.xml
index 31de16e..22104c7 100644
--- a/lang/py/ivysettings.xml
+++ b/lang/py/ivysettings.xml
@@ -19,8 +19,8 @@
<property name="m2-pattern" value="${user.home}/.m2/repository/[organisation]/[module]/[revision]/[module]-[revision](-[classifier]).[ext]" override="false" />
<resolvers>
<chain name="repos">
- <ibiblio name="central" m2compatible="true"/>
- <ibiblio name="apache-snapshots" m2compatible="true" root="https://repository.apache.org/content/groups/snapshots"/>
+ <ibiblio name="central" m2compatible="true"/>
+ <ibiblio name="apache-snapshots" m2compatible="true" root="https://repository.apache.org/content/groups/snapshots"/>
<filesystem name="local-maven2" m2compatible="true"> <!-- needed when building non-snapshot version for release -->
<artifact pattern="${m2-pattern}"/>
<ivy pattern="${m2-pattern}"/>
http://git-wip-us.apache.org/repos/asf/avro/blob/da22ffcb/lang/py3/scripts/avro
----------------------------------------------------------------------
diff --git a/lang/py3/scripts/avro b/lang/py3/scripts/avro
old mode 100644
new mode 100755