You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by ma...@apache.org on 2013/04/13 06:43:26 UTC
svn commit: r1467548 - in /avro/trunk: CHANGES.txt build.sh
lang/java/avro/pom.xml
lang/java/avro/src/test/java/org/apache/avro/RandomData.java
lang/php/test/InterOpTest.php lang/ruby/interop/test_interop.rb
Author: martinkl
Date: Sat Apr 13 04:43:26 2013
New Revision: 1467548
URL: http://svn.apache.org/r1467548
Log:
AVRO-1287. Add data file with deflate codec to the interoperability test suite.
Modified:
avro/trunk/CHANGES.txt
avro/trunk/build.sh
avro/trunk/lang/java/avro/pom.xml
avro/trunk/lang/java/avro/src/test/java/org/apache/avro/RandomData.java
avro/trunk/lang/php/test/InterOpTest.php
avro/trunk/lang/ruby/interop/test_interop.rb
Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1467548&r1=1467547&r2=1467548&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Sat Apr 13 04:43:26 2013
@@ -22,6 +22,9 @@ Trunk (not yet released)
AVRO-1259. Java: Improve Trevni's encoding of sparse columns.
(cutting)
+ AVRO-1287. Add data file with deflate codec to the interoperability
+ test suite. (martinkl)
+
BUG FIXES
AVRO-1266. Java: Fix mapred.AvroMultipleOutputs to support multiple
Modified: avro/trunk/build.sh
URL: http://svn.apache.org/viewvc/avro/trunk/build.sh?rev=1467548&r1=1467547&r2=1467548&view=diff
==============================================================================
--- avro/trunk/build.sh (original)
+++ avro/trunk/build.sh Sat Apr 13 04:43:26 2013
@@ -51,7 +51,7 @@ case "$target" in
# create interop test data
mkdir -p build/interop/data
- (cd lang/java/avro; mvn exec:java -P interop-data-generate)
+ (cd lang/java/avro; mvn -P interop-data-generate generate-resources)
(cd lang/py; ant interop-data-generate)
(cd lang/c; ./build.sh interop-data-generate)
#(cd lang/c++; make interop-data-generate)
Modified: avro/trunk/lang/java/avro/pom.xml
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/pom.xml?rev=1467548&r1=1467547&r2=1467548&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/pom.xml (original)
+++ avro/trunk/lang/java/avro/pom.xml Sat Apr 13 04:43:26 2013
@@ -78,25 +78,49 @@
<profiles>
<profile>
<id>interop-data-generate</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
<build>
- <pluginManagement>
- <plugins>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>exec-maven-plugin</artifactId>
- <!-- default configuration is to generate random data for interop tests -->
- <configuration>
- <mainClass>org.apache.avro.RandomData</mainClass>
- <classpathScope>test</classpathScope>
- <arguments>
- <argument>../../../share/test/schemas/interop.avsc</argument>
- <argument>../../../build/interop/data/java.avro</argument>
- <argument>100</argument>
- </arguments>
- </configuration>
- </plugin>
- </plugins>
- </pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <executions>
+ <!-- Generate random data for interop tests, using null codec -->
+ <execution>
+ <id>interop-generate-null-codec</id>
+ <phase>generate-resources</phase>
+ <configuration>
+ <mainClass>org.apache.avro.RandomData</mainClass>
+ <classpathScope>test</classpathScope>
+ <arguments>
+ <argument>../../../share/test/schemas/interop.avsc</argument>
+ <argument>../../../build/interop/data/java.avro</argument>
+ <argument>100</argument>
+ </arguments>
+ </configuration>
+ <goals><goal>java</goal></goals>
+ </execution>
+ <!-- Generate random data for interop tests, using deflate codec -->
+ <execution>
+ <id>interop-generate-deflate-codec</id>
+ <phase>generate-resources</phase>
+ <configuration>
+ <mainClass>org.apache.avro.RandomData</mainClass>
+ <classpathScope>test</classpathScope>
+ <arguments>
+ <argument>../../../share/test/schemas/interop.avsc</argument>
+ <argument>../../../build/interop/data/java_deflate.avro</argument>
+ <argument>100</argument>
+ <argument>deflate</argument>
+ </arguments>
+ </configuration>
+ <goals><goal>java</goal></goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
</build>
</profile>
</profiles>
Modified: avro/trunk/lang/java/avro/src/test/java/org/apache/avro/RandomData.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/test/java/org/apache/avro/RandomData.java?rev=1467548&r1=1467547&r2=1467548&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/test/java/org/apache/avro/RandomData.java (original)
+++ avro/trunk/lang/java/avro/src/test/java/org/apache/avro/RandomData.java Sat Apr 13 04:43:26 2013
@@ -25,6 +25,7 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
+import org.apache.avro.file.CodecFactory;
import org.apache.avro.file.DataFileWriter;
import org.apache.avro.generic.GenericArray;
import org.apache.avro.generic.GenericData;
@@ -123,14 +124,15 @@ public class RandomData implements Itera
}
public static void main(String[] args) throws Exception {
- if(args.length != 3) {
- System.out.println("Usage: RandomData <schemafile> <outputfile> <count>");
+ if(args.length < 3 || args.length > 4) {
+ System.out.println("Usage: RandomData <schemafile> <outputfile> <count> [codec]");
System.exit(-1);
}
Schema sch = Schema.parse(new File(args[0]));
DataFileWriter<Object> writer =
- new DataFileWriter<Object>(new GenericDatumWriter<Object>())
- .create(sch, new File(args[1]));
+ new DataFileWriter<Object>(new GenericDatumWriter<Object>());
+ writer.setCodec(CodecFactory.fromString(args.length >= 4 ? args[3] : "null"));
+ writer.create(sch, new File(args[1]));
try {
for (Object datum : new RandomData(sch, Integer.parseInt(args[2]))) {
writer.append(datum);
Modified: avro/trunk/lang/php/test/InterOpTest.php
URL: http://svn.apache.org/viewvc/avro/trunk/lang/php/test/InterOpTest.php?rev=1467548&r1=1467547&r2=1467548&view=diff
==============================================================================
--- avro/trunk/lang/php/test/InterOpTest.php (original)
+++ avro/trunk/lang/php/test/InterOpTest.php Sat Apr 13 04:43:26 2013
@@ -37,8 +37,13 @@ class InterOpTest extends PHPUnit_Framew
$data_files = array();
if (!($dh = opendir($data_dir)))
die("Could not open data dir '$data_dir'\n");
+
+ /* TODO This currently only tries to read files of the form 'language.avro',
+ * but not 'language_deflate.avro' as the PHP implementation is not yet
+ * able to read deflate data files. When deflate support is added, change
+ * this to match *.avro. */
while ($file = readdir($dh))
- if (0 < preg_match('/.*\.avro$/', $file))
+ if (0 < preg_match('/^[a-z]+\.avro$/', $file))
$data_files []= join(DIRECTORY_SEPARATOR, array($data_dir, $file));
closedir($dh);
Modified: avro/trunk/lang/ruby/interop/test_interop.rb
URL: http://svn.apache.org/viewvc/avro/trunk/lang/ruby/interop/test_interop.rb?rev=1467548&r1=1467547&r2=1467548&view=diff
==============================================================================
--- avro/trunk/lang/ruby/interop/test_interop.rb (original)
+++ avro/trunk/lang/ruby/interop/test_interop.rb Sat Apr 13 04:43:26 2013
@@ -23,7 +23,10 @@ class TestInterop < Test::Unit::TestCase
HERE = File.expand_path(File.dirname(__FILE__))
SHARE = HERE + '/../../../share'
SCHEMAS = SHARE + '/test/schemas'
- Dir[HERE + '/../../../build/interop/data/*'].each do |fn|
+
+ # TODO this currently ignores data files with deflate codec.
+ # Remove this restriction when deflate support is added.
+ Dir[HERE + '/../../../build/interop/data/*'].reject{|fn| fn =~ /_deflate/ }.each do |fn|
define_method("test_read_#{File.basename(fn, 'avro')}") do
projection = Avro::Schema.parse(File.read(SCHEMAS+'/interop.avsc'))