You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by bi...@apache.org on 2012/05/12 03:45:12 UTC
svn commit: r1337429 - in /pig/trunk: ./ src/org/apache/pig/
src/org/apache/pig/builtin/ test/org/apache/pig/
test/org/apache/pig/backend/ test/org/apache/pig/backend/hadoop/
test/org/apache/pig/backend/hadoop/executionengine/
test/org/apache/pig/backe...
Author: billgraham
Date: Sat May 12 01:45:11 2012
New Revision: 1337429
URL: http://svn.apache.org/viewvc?rev=1337429&view=rev
Log:
PIG-2547: Easier UDFs: Convenient EvalFunc super-classes (billgraham, dvryaboy)
Added:
pig/trunk/src/org/apache/pig/ExceptionalFunction.java
pig/trunk/src/org/apache/pig/Function.java
pig/trunk/src/org/apache/pig/PrimitiveEvalFunc.java
pig/trunk/src/org/apache/pig/TypedOutputEvalFunc.java
pig/trunk/src/org/apache/pig/builtin/FunctionWrapperEvalFunc.java
pig/trunk/test/org/apache/pig/TestPrimitiveEvalFunc.java
pig/trunk/test/org/apache/pig/TestTypedOutputEvalFunc.java
pig/trunk/test/org/apache/pig/backend/
pig/trunk/test/org/apache/pig/backend/hadoop/
pig/trunk/test/org/apache/pig/backend/hadoop/executionengine/
pig/trunk/test/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/
pig/trunk/test/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/TestInputSizeReducerEstimator.java
pig/trunk/test/org/apache/pig/builtin/TestFunctionWrapperEvalFunc.java
Modified:
pig/trunk/CHANGES.txt
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1337429&r1=1337428&r2=1337429&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Sat May 12 01:45:11 2012
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
IMPROVEMENTS
+PIG-2547: Easier UDFs: Convenient EvalFunc super-classes (billgraham, dvryaboy)
+
PIG-2639: Utils.getSchemaFromString should automatically give name to all types, but fails on boolean (jcoveney)
PIG-2583: Add Grunt command to list the statements in cache (xalan via daijy)
@@ -34,6 +36,14 @@ PIG-2680: TOBAG output schema reporting
PIG-2685: Fix error in EvalFunc ctor when implementing Algebraic UDF whose return type is parameterized (andy schlaikjer via jcoveney)
+PIG-2664: Allow PPNL impls to get more job info during the run (billgraham)
+
+PIG-2663: Expose helpful ScriptState methods (billgraham)
+
+PIG-2660: PPNL notified of plan before it gets executed (billgraham)
+
+PIG-2574: Make reducer estimator plugable (billgraham)
+
PIG-2677: Add target to build.xml to generate clover summary reports (gates)
PIG-2650: Convenience mock Loader and Storer to simplify unit testing of Pig scripts (julien)
Added: pig/trunk/src/org/apache/pig/ExceptionalFunction.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/ExceptionalFunction.java?rev=1337429&view=auto
==============================================================================
--- pig/trunk/src/org/apache/pig/ExceptionalFunction.java (added)
+++ pig/trunk/src/org/apache/pig/ExceptionalFunction.java Sat May 12 01:45:11 2012
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig;
+
+/**
+ * An interface that captures a unit of work against an item where an exception might be thrown.
+ *
+ * @param <S> The argument type for the function.
+ * @param <T> The return type for the function.
+ * @param <E> The exception type that the function throws.
+ *
+ */
+public interface ExceptionalFunction<S, T, E extends Exception> {
+
+ /**
+ * Performs a unit of work on item, possibly throwing {@code E} in the process.
+ *
+ * @param item The item to perform work against.
+ * @return The result of the computation.
+ * @throws E if there was a problem performing the work.
+ */
+ public T apply(S item) throws E;
+}
\ No newline at end of file
Added: pig/trunk/src/org/apache/pig/Function.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/Function.java?rev=1337429&view=auto
==============================================================================
--- pig/trunk/src/org/apache/pig/Function.java (added)
+++ pig/trunk/src/org/apache/pig/Function.java Sat May 12 01:45:11 2012
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig;
+
+/**
+ * A convenience typedef that ties into both google's {@code Function} and
+ * {@code ExceptionalFunction}.
+ *
+ * @param <S> The argument type for the function.
+ * @param <T> The return type for the function.
+ *
+ */
+public interface Function<S, T>
+ extends ExceptionalFunction<S, T, RuntimeException>, com.google.common.base.Function<S, T> {
+
+ @Override
+ public T apply(S item);
+}
Added: pig/trunk/src/org/apache/pig/PrimitiveEvalFunc.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/PrimitiveEvalFunc.java?rev=1337429&view=auto
==============================================================================
--- pig/trunk/src/org/apache/pig/PrimitiveEvalFunc.java (added)
+++ pig/trunk/src/org/apache/pig/PrimitiveEvalFunc.java Sat May 12 01:45:11 2012
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig;
+
+import org.apache.pig.data.Tuple;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Base class for simple Pig UDFs that are functions of primitive types IN to OUT. Handles
+ * marshalling objects, basic error checking, etc. Extend this class and implement the
+ * <pre>OUT exec(IN input)</pre> method when writting a UDF that operates on only the first input
+ * (of expected type IN) from the Tuple.
+ */
+public abstract class PrimitiveEvalFunc<IN, OUT> extends TypedOutputEvalFunc<OUT> {
+
+ protected Class<IN> inTypeClass = null;
+
+ public Class<IN> getInputTypeClass() { return inTypeClass; }
+
+ @SuppressWarnings("unchecked")
+ public PrimitiveEvalFunc() {
+ List<?> typeArgs = getTypeArguments(PrimitiveEvalFunc.class, getClass());
+ inTypeClass = (Class<IN>) typeArgs.get(0);
+ outTypeClass = (Class<OUT>) typeArgs.get(1);
+ }
+
+ @SuppressWarnings("unchecked")
+ public PrimitiveEvalFunc(Class inTypeClass, Class outTypeClass) {
+ this.inTypeClass = inTypeClass;
+ this.outTypeClass = outTypeClass;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public OUT exec(Tuple tuple) throws IOException {
+ verifyUdfInput(getCounterGroup(), tuple, 1);
+
+ IN input = (IN) tuple.get(0);
+ if (input == null) {
+ // Default behavior of null input should be null output.
+ return null;
+ }
+
+ return exec(input);
+ }
+
+ public abstract OUT exec(IN input) throws IOException;
+}
Added: pig/trunk/src/org/apache/pig/TypedOutputEvalFunc.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/TypedOutputEvalFunc.java?rev=1337429&view=auto
==============================================================================
--- pig/trunk/src/org/apache/pig/TypedOutputEvalFunc.java (added)
+++ pig/trunk/src/org/apache/pig/TypedOutputEvalFunc.java Sat May 12 01:45:11 2012
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig;
+
+import com.google.common.collect.Maps;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.tools.pigstats.PigStatusReporter;
+
+import java.io.IOException;
+import java.lang.reflect.*;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Base class for Pig UDFs that are functions from Tuples to generic type OUT. Handles marshalling
+ * objects, basic error checking, etc. Also infers outputSchema and provides a function to verify
+ * the input Tuple.
+ * <P></P>
+ * Extend this class and implement the <pre>OUT exec(Tuple input)</pre> method when writting a UDF
+ * that operates on multiple inputs from the Tuple.
+ */
+public abstract class TypedOutputEvalFunc<OUT> extends EvalFunc<OUT> {
+
+ // Used to implement outputSchema below.
+ protected Class<OUT> outTypeClass = null;
+
+ public Class<OUT> getOutputTypeClass() { return outTypeClass; }
+
+ @SuppressWarnings("unchecked")
+ public TypedOutputEvalFunc() {
+ outTypeClass = (Class<OUT>) getTypeArguments(TypedOutputEvalFunc.class, getClass()).get(0);
+ }
+
+ // Increment Hadoop counters for bad inputs which are either null or too small.
+ protected void verifyInput(Tuple input, int minimumSize) throws IOException {
+ verifyUdfInput(getCounterGroup(), input, minimumSize);
+ }
+
+ /**
+ * Incremented counters will use this as the counter group. Typically this works fine, since
+ * the subclass name is enough to identify the UDF. In some cases though (i.e. a UDF wrapper that
+ * is a facade to a number of different transformation functions), a more specific group name is
+ * needed.
+ */
+ protected String getCounterGroup() {
+ return getClass().getName();
+ }
+
+ /**
+ * Get the actual type arguments a child class has used to extend a generic base class.
+ *
+ * @param baseClass the base class
+ * @param childClass the child class
+ * @return a list of the raw classes for the actual type arguments.
+ */
+ @SuppressWarnings("unchecked")
+ protected static <T> List<Class<?>> getTypeArguments(Class<T> baseClass,
+ Class<? extends T> childClass) {
+ Map<Type, Type> resolvedTypes = Maps.newHashMap();
+ Type type = childClass;
+ // start walking up the inheritance hierarchy until we hit baseClass
+ while (!getClass(type).equals(baseClass)) {
+ if (type instanceof Class) {
+ // there is no useful information for us in raw types, so just keep going.
+ type = ((Class) type).getGenericSuperclass();
+ } else {
+ ParameterizedType parameterizedType = (ParameterizedType) type;
+ Class<?> rawType = (Class) parameterizedType.getRawType();
+
+ Type[] actualTypeArguments = parameterizedType.getActualTypeArguments();
+ TypeVariable<?>[] typeParameters = rawType.getTypeParameters();
+ for (int i = 0; i < actualTypeArguments.length; i++) {
+ resolvedTypes.put(typeParameters[i], actualTypeArguments[i]);
+ }
+
+ if (!rawType.equals(baseClass)) {
+ type = rawType.getGenericSuperclass();
+ }
+ }
+ }
+
+ // finally, for each actual type argument provided to baseClass, determine (if possible)
+ // the raw class for that type argument.
+ Type[] actualTypeArguments;
+ if (type instanceof Class) {
+ actualTypeArguments = ((Class) type).getTypeParameters();
+ } else {
+ actualTypeArguments = ((ParameterizedType) type).getActualTypeArguments();
+ }
+ List<Class<?>> typeArgumentsAsClasses = new ArrayList<Class<?>>();
+ // resolve types by chasing down type variables.
+ for (Type baseType : actualTypeArguments) {
+ while (resolvedTypes.containsKey(baseType)) {
+ baseType = resolvedTypes.get(baseType);
+ }
+ typeArgumentsAsClasses.add(getClass(baseType));
+ }
+ return typeArgumentsAsClasses;
+ }
+
+ /**
+ * Get the underlying class for a type, or null if the type is a variable type.
+ *
+ * @param type the type
+ * @return the underlying class
+ */
+ @SuppressWarnings("unchecked")
+ private static Class<?> getClass(Type type) {
+ if (type instanceof Class) {
+ return (Class) type;
+ } else if (type instanceof ParameterizedType) {
+ return getClass(((ParameterizedType) type).getRawType());
+ } else if (type instanceof GenericArrayType) {
+ Type componentType = ((GenericArrayType) type).getGenericComponentType();
+ Class<?> componentClass = getClass(componentType);
+ if (componentClass != null) {
+ return Array.newInstance(componentClass, 0).getClass();
+ } else {
+ return null;
+ }
+ } else {
+ return null;
+ }
+ }
+
+ /** Increment Hadoop counters for bad inputs which are either null or too small.
+ *
+ * @param klass the name of the calling class, for recording purposes
+ * @param input the tuple passed to the UDF.
+ * @param minimumSize the minimum size required of the tuple.
+ */
+ protected static void verifyUdfInput(String klass, Tuple input, int minimumSize) throws IOException {
+ if (input == null) {
+ safeIncrCounter(klass, "NullInput", 1L);
+ throw new IOException("Null input to UDF " + klass);
+ } else if (input.size() < minimumSize) {
+ String reason = "TooFewArguments_Got_" + input.size() + "_NeededAtLeast_" + minimumSize;
+ safeIncrCounter(klass, reason, 1L);
+ throw new IOException("Not enough arguments to " + klass + ": got " + input.size() +
+ ", expected at least " + minimumSize);
+ } else {
+ safeIncrCounter(klass, "ValidInput", 1L);
+ }
+ }
+
+ protected static void safeIncrCounter(String group, String name, Long increment) {
+ Counter counter = PigStatusReporter.getInstance().getCounter(group, name);
+ if (counter != null) {
+ counter.increment(increment);
+ }
+ }
+}
Added: pig/trunk/src/org/apache/pig/builtin/FunctionWrapperEvalFunc.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/builtin/FunctionWrapperEvalFunc.java?rev=1337429&view=auto
==============================================================================
--- pig/trunk/src/org/apache/pig/builtin/FunctionWrapperEvalFunc.java (added)
+++ pig/trunk/src/org/apache/pig/builtin/FunctionWrapperEvalFunc.java Sat May 12 01:45:11 2012
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.builtin;
+
+import org.apache.pig.ExceptionalFunction;
+import org.apache.pig.PrimitiveEvalFunc;
+import org.apache.pig.impl.PigContext;
+
+import java.io.IOException;
+import java.lang.reflect.ParameterizedType;
+import java.lang.reflect.Type;
+import java.util.HashMap;
+
+/**
+ * EvalFunc that wraps an implementation of the Function interface, which is passed as a String
+ * in the constructor. When resolving the Function class, the Pig UDF package import list is used.
+ * <P>
+ * The Function must have a default no-arg constructor, which will be used. For Functions that
+ * take args in the constructor, initialize the function in a subclass of this one and call
+ * <code>super(function)</code>.
+ * <P>
+ * Example: <code>DEFINE myUdf FunctionWrapperEvalFunc('MyFunction')</code>
+ *
+ */
+public class FunctionWrapperEvalFunc extends PrimitiveEvalFunc<Object, Object> {
+
+ // cache the types we resolve to limit reflection
+ private static HashMap<Class, Type[]> resolvedTypes = new HashMap<Class, Type[]>();
+
+ private ExceptionalFunction function;
+ private String counterGroup;
+
+ /**
+ * Takes the class name of a Function, initializes it using the default constructor and passes
+ * it to FunctionWrapperEvalFunc(ExceptionalFunction function). Functions must implement either
+ * com.google.common.base.Function or ExceptionalFunction.
+ * @param functionClassName function class to initialize
+ */
+ public FunctionWrapperEvalFunc(String functionClassName)
+ throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException,
+ InstantiationException, IOException {
+
+ this(initializeFunction(functionClassName));
+ }
+
+ /**
+ * Determines the input and output types of the Function and initializes the superclass.
+ * Subclass and call this constructor if a Function with a non-default constructor is required.
+ * @param function Function to be used by the UDF.
+ */
+ protected FunctionWrapperEvalFunc(com.google.common.base.Function function)
+ throws IOException, ClassNotFoundException, NoSuchMethodException {
+
+ this((ExceptionalFunction) new GoogleFunctionBridge(function));
+ }
+
+ /**
+ * Determines the input and output types of the Function and initializes the superclass.
+ * Subclass and call this constructor if a Function with a non-default constructor is required.
+ * @param function Function to be used by the UDF.
+ */
+ protected FunctionWrapperEvalFunc(ExceptionalFunction function)
+ throws IOException, ClassNotFoundException, NoSuchMethodException {
+
+ super(getFunctionInClass(function), getFunctionOutClass(function));
+ this.function = function;
+
+ String functionName = (function instanceof GoogleFunctionBridge) ?
+ ((GoogleFunctionBridge)function).getWrappedFunction().getClass().getSimpleName() :
+ function.getClass().getSimpleName();
+ this.counterGroup = getClass().getName() + ":" + functionName;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public Object exec(Object input) throws IOException {
+ try {
+ return function.apply(input);
+ } catch (Exception e) {
+ safeIncrCounter(getCounterGroup(), e.getClass().getCanonicalName(), 1L);
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ protected String getCounterGroup() {
+ return this.counterGroup;
+ }
+
+ private static Class getFunctionInClass(ExceptionalFunction functionClassName)
+ throws ClassNotFoundException, NoSuchMethodException, IOException {
+ return getFunctionTypeClass(functionClassName, 0);
+ }
+
+ private static Class getFunctionOutClass(ExceptionalFunction functionClassName)
+ throws ClassNotFoundException, NoSuchMethodException, IOException {
+ return getFunctionTypeClass(functionClassName, 1);
+ }
+
+ /**
+ * For a given class that implements the parameterized interface <code>ExceptionalFunction</code>,
+ * return the type class at the <code>index</code> position. If the Function class, is
+ * <code>GoogleFunctionBridge</code>, return the type class for the wrapped function.
+ */
+ private static Class getFunctionTypeClass(ExceptionalFunction function, int index)
+ throws ClassNotFoundException, NoSuchMethodException, IOException {
+
+ Class clazz;
+ Class expectedInterface;
+ if (function instanceof GoogleFunctionBridge) {
+ clazz = ((GoogleFunctionBridge) function).getWrappedFunction().getClass();
+ expectedInterface = com.google.common.base.Function.class;
+ }
+ else {
+ clazz = function.getClass();
+ expectedInterface = ExceptionalFunction.class;
+ }
+
+ // check the cache
+ if (resolvedTypes.containsKey(clazz)) {
+ return (Class)resolvedTypes.get(clazz)[index];
+ }
+
+ Type[] interfaceTypes = clazz.getGenericInterfaces();
+ for (Type interfaceType : interfaceTypes) {
+ ParameterizedType parameterizedType = (ParameterizedType)interfaceType;
+ if (expectedInterface.isAssignableFrom((Class) parameterizedType.getRawType())) {
+ Type[] types = parameterizedType.getActualTypeArguments();
+ resolvedTypes.put(clazz, types);
+ return (Class)types[index];
+ }
+ }
+
+ throw new NoSuchMethodException("Unrecognized function class passed: "
+ + clazz.getClass() + ". Function must implement either "
+ + com.google.common.base.Function.class.getName() + " or " +
+ ExceptionalFunction.class.getName());
+ }
+
+ @SuppressWarnings("unchecked")
+ private static ExceptionalFunction initializeFunction(String functionClassName)
+ throws IOException, IllegalAccessException, InstantiationException {
+
+ Object functionObject = PigContext.resolveClassName(functionClassName).newInstance();
+
+ if (functionObject instanceof ExceptionalFunction) {
+ return (ExceptionalFunction) functionObject;
+ }
+ else if (functionObject instanceof com.google.common.base.Function) {
+ return new GoogleFunctionBridge((com.google.common.base.Function)functionObject);
+ }
+
+ throw new InstantiationException("Unrecognized function class passed: "
+ + functionObject.getClass() + ". Function must implement either "
+ + com.google.common.base.Function.class.getName() + " or " +
+ ExceptionalFunction.class.getName());
+ }
+
+ /**
+ * Used so we can handle both Google's Function as well as an Pig's ExceptionalFunction.
+ */
+ private static class GoogleFunctionBridge<S, T> implements org.apache.pig.Function {
+ private com.google.common.base.Function<S, T> function;
+
+ private GoogleFunctionBridge(com.google.common.base.Function<S, T> function) {
+ this.function = function;
+ }
+
+ public com.google.common.base.Function getWrappedFunction() { return function; }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public T apply(Object item) {
+ return function.apply((S)item);
+ }
+ }
+}
\ No newline at end of file
Added: pig/trunk/test/org/apache/pig/TestPrimitiveEvalFunc.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/TestPrimitiveEvalFunc.java?rev=1337429&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/TestPrimitiveEvalFunc.java (added)
+++ pig/trunk/test/org/apache/pig/TestPrimitiveEvalFunc.java Sat May 12 01:45:11 2012
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig;
+
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestPrimitiveEvalFunc {
+
+ private static class StringStringFunc extends PrimitiveEvalFunc<String, String> {
+ @Override
+ public String exec(String input) throws IOException {
+ return "foo";
+ }
+ };
+
+ private static class StringIntegerFunc extends PrimitiveEvalFunc<String, Integer> {
+ @Override
+ public Integer exec(String input) throws IOException {
+ return 1;
+ }
+ };
+
+ private static class IntegerFloatFunc extends PrimitiveEvalFunc<Integer, Float> {
+ @Override
+ public Float exec(Integer i) throws IOException {
+ return 1.0f;
+ }
+ };
+
+ @Test
+ public final void testGetOutputTypes() throws IOException {
+ StringStringFunc udf1 = new StringStringFunc();
+ StringIntegerFunc udf2 = new StringIntegerFunc();
+ IntegerFloatFunc udf3 = new IntegerFloatFunc();
+
+ assertEquals(String.class, udf1.getOutputTypeClass());
+ assertEquals(String.class, udf1.getInputTypeClass());
+
+ assertEquals(Integer.class, udf2.getOutputTypeClass());
+ assertEquals(String.class, udf2.getInputTypeClass());
+
+ assertEquals(Float.class, udf3.getOutputTypeClass());
+ assertEquals(Integer.class, udf3.getInputTypeClass());
+ }
+}
Added: pig/trunk/test/org/apache/pig/TestTypedOutputEvalFunc.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/TestTypedOutputEvalFunc.java?rev=1337429&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/TestTypedOutputEvalFunc.java (added)
+++ pig/trunk/test/org/apache/pig/TestTypedOutputEvalFunc.java Sat May 12 01:45:11 2012
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig;
+
+import org.apache.pig.data.Tuple;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestTypedOutputEvalFunc {
+
+ private static class StringFunc extends TypedOutputEvalFunc<String> {
+ @Override
+ public String exec(Tuple input) throws IOException {
+ return "foo";
+ }
+ };
+
+ private static class IntegerFunc extends TypedOutputEvalFunc<Integer> {
+ @Override
+ public Integer exec(Tuple input) throws IOException {
+ return 1;
+ }
+ };
+
+ private static class FloatFunc extends TypedOutputEvalFunc<Float> {
+ @Override
+ public Float exec(Tuple input) throws IOException {
+ return 1.0f;
+ }
+ };
+
+ private static final TypedOutputEvalFunc<String> stringFunc = new TypedOutputEvalFunc<String>() {
+ @Override
+ public String exec(Tuple input) throws IOException {
+ return "foo";
+ }
+ };
+
+ private static final TypedOutputEvalFunc<Integer> intFunc = new TypedOutputEvalFunc<Integer>() {
+ @Override
+ public Integer exec(Tuple input) throws IOException {
+ return 1;
+ }
+ };
+
+ private static final TypedOutputEvalFunc<Float> floatFunc = new TypedOutputEvalFunc<Float>() {
+ @Override
+ public Float exec(Tuple input) throws IOException {
+ return 1.0f;
+ }
+ };
+
+ @Test
+ public final void testGetOutputTypes() throws IOException {
+ // Verify that type inference works for declared as well as anonymous classes.
+ assertEquals(String.class, stringFunc.getOutputTypeClass());
+ assertEquals(Integer.class, intFunc.getOutputTypeClass());
+ assertEquals(Float.class, floatFunc.getOutputTypeClass());
+
+ assertEquals(String.class, new StringFunc().getOutputTypeClass());
+ assertEquals(Integer.class, new IntegerFunc().getOutputTypeClass());
+ assertEquals(Float.class, new FloatFunc().getOutputTypeClass());
+ }
+}
Added: pig/trunk/test/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/TestInputSizeReducerEstimator.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/TestInputSizeReducerEstimator.java?rev=1337429&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/TestInputSizeReducerEstimator.java (added)
+++ pig/trunk/test/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/TestInputSizeReducerEstimator.java Sat May 12 01:45:11 2012
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.backend.hadoop.executionengine.mapReduceLayer;
+
+import com.google.common.collect.Lists;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.pig.LoadFunc;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLoad;
+import org.apache.pig.builtin.PigStorage;
+import org.apache.pig.test.PigStorageWithStatistics;
+import org.apache.pig.test.TestJobControlCompiler;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestInputSizeReducerEstimator {
+
+ private static final Configuration CONF = new Configuration();
+
+ @Test
+ public void testGetInputSizeFromFs() throws Exception {
+ long size = 2L * 1024 * 1024 * 1024;
+ Assert.assertEquals(size, InputSizeReducerEstimator.getTotalInputFileSize(
+ CONF, Lists.newArrayList(createPOLoadWithSize(size, new PigStorage())),
+ new org.apache.hadoop.mapreduce.Job(CONF)));
+
+ Assert.assertEquals(size, InputSizeReducerEstimator.getTotalInputFileSize(
+ CONF,
+ Lists.newArrayList(createPOLoadWithSize(size, new PigStorageWithStatistics())),
+ new org.apache.hadoop.mapreduce.Job(CONF)));
+
+ Assert.assertEquals(size * 2, InputSizeReducerEstimator.getTotalInputFileSize(
+ CONF,
+ Lists.newArrayList(
+ createPOLoadWithSize(size, new PigStorage()),
+ createPOLoadWithSize(size, new PigStorageWithStatistics())),
+ new org.apache.hadoop.mapreduce.Job(CONF)));
+ }
+
+ @Test
+ public void testGetInputSizeFromLoader() throws Exception {
+ long size = 2L * 1024 * 1024 * 1024;
+ Assert.assertEquals(size, InputSizeReducerEstimator.getInputSizeFromLoader(
+ createPOLoadWithSize(size, new PigStorageWithStatistics()),
+ new org.apache.hadoop.mapreduce.Job(CONF)));
+ }
+
+ private static POLoad createPOLoadWithSize(long size, LoadFunc loadFunc) throws Exception {
+ return TestJobControlCompiler.createPOLoadWithSize(size, loadFunc);
+ }
+}
Added: pig/trunk/test/org/apache/pig/builtin/TestFunctionWrapperEvalFunc.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/builtin/TestFunctionWrapperEvalFunc.java?rev=1337429&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/builtin/TestFunctionWrapperEvalFunc.java (added)
+++ pig/trunk/test/org/apache/pig/builtin/TestFunctionWrapperEvalFunc.java Sat May 12 01:45:11 2012
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.builtin;
+
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+public class TestFunctionWrapperEvalFunc {
+
+ public static class StringStringFunction implements org.apache.pig.Function<String, String> {
+ @Override
+ public String apply(String s) {
+ return "foo";
+ }
+ };
+
+ public static class StringIntegerFunction implements org.apache.pig.Function<String, Integer> {
+ @Override
+ public Integer apply(String s) {
+ return new Integer(s);
+ }
+ };
+
+ public static class IntegerFloatFunction implements com.google.common.base.Function<Integer, Float> {
+ @Override
+ public Float apply(Integer integer) {
+ return new Float(integer);
+ }
+ };
+
+ public static class ExceptionalIntegerFloatFunction
+ implements org.apache.pig.ExceptionalFunction<Integer, Float, MockException> {
+ @Override
+ public Float apply(Integer integer) throws MockException {
+ if (integer > 10) {
+ throw new MockException();
+ }
+ return new Float(integer);
+ }
+ };
+
+ public static class MockException extends Exception {}
+
+ @Test
+ public final void testStringToString() throws IOException, ClassNotFoundException,
+ NoSuchMethodException, InstantiationException, IllegalAccessException {
+ FunctionWrapperEvalFunc udf = new FunctionWrapperEvalFunc(StringStringFunction.class.getName());
+
+ assertEquals(String.class, udf.getInputTypeClass());
+ assertEquals(String.class, udf.getOutputTypeClass());
+ assertEquals("foo", udf.exec("xyz"));
+ }
+
+ @Test
+ public final void testStringToInteger() throws IOException, ClassNotFoundException,
+ NoSuchMethodException, InstantiationException, IllegalAccessException {
+ FunctionWrapperEvalFunc udf = new FunctionWrapperEvalFunc(StringIntegerFunction.class.getName());
+
+ assertEquals(String.class, udf.getInputTypeClass());
+ assertEquals(Integer.class, udf.getOutputTypeClass());
+ assertEquals(4, udf.exec("4"));
+ }
+
+ @Test
+ public final void testStringToIntegerPreinitialized() throws IOException, ClassNotFoundException,
+ NoSuchMethodException, InstantiationException, IllegalAccessException {
+ org.apache.pig.ExceptionalFunction function = new StringIntegerFunction();
+ FunctionWrapperEvalFunc udf = new FunctionWrapperEvalFunc(function);
+
+ assertEquals(String.class, udf.getInputTypeClass());
+ assertEquals(Integer.class, udf.getOutputTypeClass());
+ assertEquals(4, udf.exec("4"));
+ }
+
+ @Test
+ public final void testIntegerToFloat() throws IOException, ClassNotFoundException,
+ NoSuchMethodException, InstantiationException, IllegalAccessException {
+ FunctionWrapperEvalFunc udf = new FunctionWrapperEvalFunc(IntegerFloatFunction.class.getName());
+
+ assertEquals(Integer.class, udf.getInputTypeClass());
+ assertEquals(Float.class, udf.getOutputTypeClass());
+ assertEquals(4.0f, udf.exec(new Integer(4)));
+ }
+
+ @Test
+ public final void testIntegerToFloatPreinitialized() throws IOException, ClassNotFoundException,
+ NoSuchMethodException, InstantiationException, IllegalAccessException {
+ com.google.common.base.Function function = new IntegerFloatFunction();
+ FunctionWrapperEvalFunc udf = new FunctionWrapperEvalFunc(function);
+
+ assertEquals(Integer.class, udf.getInputTypeClass());
+ assertEquals(Float.class, udf.getOutputTypeClass());
+ assertEquals(4.0f, udf.exec(new Integer(4)));
+ }
+
+ @Test
+ public final void testExceptionalIntegerToFloat() throws IOException, ClassNotFoundException,
+ NoSuchMethodException, InstantiationException, IllegalAccessException {
+ FunctionWrapperEvalFunc udf =
+ new FunctionWrapperEvalFunc(ExceptionalIntegerFloatFunction.class.getName());
+
+ assertEquals(Integer.class, udf.getInputTypeClass());
+ assertEquals(Float.class, udf.getOutputTypeClass());
+ assertEquals(4.0f, udf.exec(new Integer(4)));
+ }
+
+ @Test
+ public final void testExceptionalIntegerToFloatWithException() throws IOException, ClassNotFoundException,
+ NoSuchMethodException, InstantiationException, IllegalAccessException {
+
+ FunctionWrapperEvalFunc udf =
+ new FunctionWrapperEvalFunc(ExceptionalIntegerFloatFunction.class.getName());
+
+ assertEquals(Float.class, udf.getOutputTypeClass());
+ Exception exception = null;
+ try {
+ udf.exec(new Integer(11));
+ } catch (IOException e) {
+ exception = e;
+ }
+
+ assertNotNull(exception);
+ assertEquals(IOException.class, exception.getClass());
+ assertEquals(MockException.class, exception.getCause().getClass());
+ }
+}