You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@crunch.apache.org by jw...@apache.org on 2013/09/17 00:28:36 UTC
git commit: CRUNCH-266: AvroSpecificDeepCopier needs to use
constructor on SpecificDatumReader that takes a class. Contributed by Brian
Dougan.
Updated Branches:
refs/heads/master acdf396b5 -> 995f4d985
CRUNCH-266: AvroSpecificDeepCopier needs to use constructor on SpecificDatumReader that takes a class. Contributed by Brian Dougan.
Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/995f4d98
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/995f4d98
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/995f4d98
Branch: refs/heads/master
Commit: 995f4d9854c3008884171719e419bb9b618740cf
Parents: acdf396
Author: Josh Wills <jw...@apache.org>
Authored: Mon Sep 16 15:23:08 2013 -0700
Committer: Josh Wills <jw...@apache.org>
Committed: Mon Sep 16 15:23:08 2013 -0700
----------------------------------------------------------------------
.../crunch/types/avro/AvroDeepCopier.java | 8 +-
.../avro/AvroChildClassloaderTestRunner.java | 77 ++++++++++++++++++++
.../AvroSpecificDeepCopierClassloaderTest.java | 46 ++++++++++++
3 files changed, 126 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/crunch/blob/995f4d98/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroDeepCopier.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroDeepCopier.java b/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroDeepCopier.java
index 0fe9288..367bc9f 100644
--- a/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroDeepCopier.java
+++ b/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroDeepCopier.java
@@ -96,12 +96,12 @@ abstract class AvroDeepCopier<T> implements DeepCopier<T>, Serializable {
@Override
protected DatumWriter<T> createDatumWriter(Configuration conf) {
- return new SpecificDatumWriter<T>(getSchema());
+ return new SpecificDatumWriter<T>(valueClass);
}
@Override
protected DatumReader<T> createDatumReader(Configuration conf) {
- return new SpecificDatumReader<T>(getSchema());
+ return new SpecificDatumReader<T>(valueClass);
}
}
@@ -188,12 +188,10 @@ abstract class AvroDeepCopier<T> implements DeepCopier<T>, Serializable {
binaryEncoder.flush();
binaryDecoder = DecoderFactory.get()
.binaryDecoder(byteOutStream.toByteArray(), binaryDecoder);
- datumReader.read(target, binaryDecoder);
+ return datumReader.read(target, binaryDecoder);
} catch (Exception e) {
throw new CrunchRuntimeException("Error while deep copying avro value " + source, e);
}
-
- return target;
}
protected T createNewInstance(Class<T> targetClass) {
http://git-wip-us.apache.org/repos/asf/crunch/blob/995f4d98/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroChildClassloaderTestRunner.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroChildClassloaderTestRunner.java b/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroChildClassloaderTestRunner.java
new file mode 100644
index 0000000..a1f93b7
--- /dev/null
+++ b/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroChildClassloaderTestRunner.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch.types.avro;
+
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.ArrayList;
+import java.util.Collection;
+
+import org.junit.runners.BlockJUnit4ClassRunner;
+import org.junit.runners.model.InitializationError;
+
+public class AvroChildClassloaderTestRunner extends BlockJUnit4ClassRunner {
+
+ public AvroChildClassloaderTestRunner(Class<?> clazz) throws InitializationError {
+ super(getFromTestClassloader(clazz));
+ }
+
+ private static Class<?> getFromTestClassloader(Class<?> clazz) throws InitializationError {
+ try {
+ ClassLoader testClassLoader = new TestClassLoader();
+ return Class.forName(clazz.getName(), true, testClassLoader);
+ } catch (ClassNotFoundException e) {
+ throw new InitializationError(e);
+ }
+ }
+
+ public static class TestClassLoader extends URLClassLoader {
+
+ private static ClassLoader parentClassLoader;
+ private static URL[] crunchURLs;
+ static {
+ URLClassLoader systemClassLoader = (URLClassLoader) getSystemClassLoader();
+
+ Collection<URL> crunchURLs = new ArrayList<URL>();
+ Collection<URL> otherURLs = new ArrayList<URL>();
+ for (URL url : systemClassLoader.getURLs()) {
+ if (url.getPath().matches("^.*/crunch-?.*/.*$")) {
+ crunchURLs.add(url);
+ } else {
+ otherURLs.add(url);
+ }
+ }
+
+ TestClassLoader.crunchURLs = crunchURLs.toArray(new URL[crunchURLs.size()]);
+ parentClassLoader = new URLClassLoader(otherURLs.toArray(new URL[otherURLs.size()]), null);
+ }
+
+ public TestClassLoader() {
+ super(crunchURLs, parentClassLoader);
+ }
+
+ @Override
+ public Class<?> loadClass(String name) throws ClassNotFoundException {
+ if (name.startsWith("org.junit")) {
+ return getSystemClassLoader().loadClass(name);
+ }
+
+ return super.loadClass(name);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/crunch/blob/995f4d98/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroSpecificDeepCopierClassloaderTest.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroSpecificDeepCopierClassloaderTest.java b/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroSpecificDeepCopierClassloaderTest.java
new file mode 100644
index 0000000..aed7fce
--- /dev/null
+++ b/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroSpecificDeepCopierClassloaderTest.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch.types.avro;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotSame;
+
+import org.apache.crunch.test.Person;
+import org.apache.crunch.types.avro.AvroDeepCopier.AvroSpecificDeepCopier;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import com.google.common.collect.Lists;
+
+@RunWith(AvroChildClassloaderTestRunner.class)
+public class AvroSpecificDeepCopierClassloaderTest {
+
+ @Test
+ public void testDeepCopyWithinChildClassloader() {
+ Person person = new Person();
+ person.name = "John Doe";
+ person.age = 42;
+ person.siblingnames = Lists.<CharSequence> newArrayList();
+
+ Person deepCopyPerson = new AvroSpecificDeepCopier<Person>(Person.class, Person.SCHEMA$)
+ .deepCopy(person);
+
+ assertEquals(person, deepCopyPerson);
+ assertNotSame(person, deepCopyPerson);
+ }
+}