You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafu.apache.org by mh...@apache.org on 2014/05/26 04:52:21 UTC
[2/2] git commit: Revert "DATAFU-50 SimpleEvalFunc should extend
ContextualEvalFunc, have good lifecycle hooks"
Revert "DATAFU-50 SimpleEvalFunc should extend ContextualEvalFunc, have good lifecycle hooks"
This reverts commit a36cab397505837113a5193de52e6439bd12c831.
Multiple test failures.
Project: http://git-wip-us.apache.org/repos/asf/incubator-datafu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-datafu/commit/faab884c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-datafu/tree/faab884c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-datafu/diff/faab884c
Branch: refs/heads/master
Commit: faab884c278a3ab5197ec3cd8931f84a664f4904
Parents: c414915
Author: Matt Hayes <mh...@linkedin.com>
Authored: Sun May 25 17:56:44 2014 -0700
Committer: Matt Hayes <mh...@linkedin.com>
Committed: Sun May 25 18:23:10 2014 -0700
----------------------------------------------------------------------
.../java/datafu/pig/util/AliasableEvalFunc.java | 8 +--
.../datafu/pig/util/ContextualEvalFunc.java | 39 +-----------
.../java/datafu/pig/util/SimpleEvalFunc.java | 4 +-
.../datafu/test/pig/util/EvalFuncTests.java | 66 --------------------
.../datafu/test/pig/util/SchemaToString.java | 56 -----------------
5 files changed, 8 insertions(+), 165 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/faab884c/datafu-pig/src/main/java/datafu/pig/util/AliasableEvalFunc.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/util/AliasableEvalFunc.java b/datafu-pig/src/main/java/datafu/pig/util/AliasableEvalFunc.java
index 87bca33..ee2c3f3 100644
--- a/datafu-pig/src/main/java/datafu/pig/util/AliasableEvalFunc.java
+++ b/datafu-pig/src/main/java/datafu/pig/util/AliasableEvalFunc.java
@@ -90,12 +90,12 @@ public abstract class AliasableEvalFunc<T> extends ContextualEvalFunc<T>
}
/**
- * Hook to capture the input schema aliases
+ * A wrapper method which captures the schema and then calls getOutputSchema
*/
@Override
- public void onReady(Schema in_schema, Schema out_schema) {
- storeFieldAliases(in_schema);
- super.onReady(in_schema, out_schema);
+ public Schema outputSchema(Schema input) {
+ storeFieldAliases(input);
+ return getOutputSchema(input);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/faab884c/datafu-pig/src/main/java/datafu/pig/util/ContextualEvalFunc.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/util/ContextualEvalFunc.java b/datafu-pig/src/main/java/datafu/pig/util/ContextualEvalFunc.java
index e2b4392..c534b77 100644
--- a/datafu-pig/src/main/java/datafu/pig/util/ContextualEvalFunc.java
+++ b/datafu-pig/src/main/java/datafu/pig/util/ContextualEvalFunc.java
@@ -23,15 +23,12 @@ import java.util.Properties;
import org.apache.pig.EvalFunc;
import org.apache.pig.impl.util.UDFContext;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
/**
* An abstract class which enables UDFs to store instance properties
* on the front end which will be available on the back end.
- *
- * For example, you can override the onReady hook method to set properties
- * at front-end (i.e. at launch time) which will be available when exec() is
- * called (on the workers themselves.
+ * For example, properties may be set in the call to outputSchema(),
+ * which will be available when exec() is called.
*
* @param <T>
*/
@@ -83,36 +80,4 @@ public abstract class ContextualEvalFunc<T> extends EvalFunc<T>
private void setInstanceName(String instanceName) {
this.instanceName = instanceName;
}
-
- /**
- * Hook method, called once the input and output schema are prepared.
- *
- * Subclasses may override to set properties on the front end (i.e. at script
- * run time) that may be played with later (e.g. at execution time).
- *
- * Child classes must (a) call super.onReady(in_schema, out_schema) so
- * that the hook chains, and (b) not mess with the schema.
- *
- * @param in_schema input schema
- * @param out_schema output schema
- */
- protected void onReady(Schema in_schema, Schema out_schema)
- {
- /* ze goggles! zey do nussing! */
- }
-
- /**
- * Override outputSchema only to add the onReady hook method. In all
- * other respects delegates to the superclass outputSchema preparation.
- *
- * @param in_schema input schema
- * @return call to super.outputSchema
- */
- @Override
- public Schema outputSchema(Schema in_schema)
- {
- Schema out_schema = super.outputSchema(in_schema);
- onReady(in_schema, out_schema);
- return out_schema;
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/faab884c/datafu-pig/src/main/java/datafu/pig/util/SimpleEvalFunc.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/util/SimpleEvalFunc.java b/datafu-pig/src/main/java/datafu/pig/util/SimpleEvalFunc.java
index 80bbce9..2d262b4 100644
--- a/datafu-pig/src/main/java/datafu/pig/util/SimpleEvalFunc.java
+++ b/datafu-pig/src/main/java/datafu/pig/util/SimpleEvalFunc.java
@@ -23,7 +23,7 @@ import java.io.IOException;
import java.lang.reflect.Method;
import java.lang.reflect.Type;
-import datafu.pig.util.ContextualEvalFunc;
+import org.apache.pig.EvalFunc;
import org.apache.pig.data.DataType;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.logicalLayer.FrontendException;
@@ -98,7 +98,7 @@ import org.apache.pig.impl.logicalLayer.schema.Schema;
*/
-public abstract class SimpleEvalFunc<T> extends ContextualEvalFunc<T>
+public abstract class SimpleEvalFunc<T> extends EvalFunc<T>
{
// TODO Add support for other UDF types (e.g., FilterFunc)
// TODO Algebraic EvalFuncs
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/faab884c/datafu-pig/src/test/java/datafu/test/pig/util/EvalFuncTests.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/test/java/datafu/test/pig/util/EvalFuncTests.java b/datafu-pig/src/test/java/datafu/test/pig/util/EvalFuncTests.java
deleted file mode 100644
index b46dd1b..0000000
--- a/datafu-pig/src/test/java/datafu/test/pig/util/EvalFuncTests.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.test.pig.util;
-
-import org.adrianwalker.multilinestring.Multiline;
-import org.apache.pig.pigunit.PigTest;
-import org.testng.annotations.Test;
-import datafu.test.pig.PigTests;
-import junit.framework.Assert;
-
-import datafu.test.pig.util.SchemaToString;
-
-public class EvalFuncTests extends PigTests
-{
-
- /**
-
-
- define SchemaToString datafu.test.pig.util.SchemaToString();
-
- data = LOAD 'input' AS (city:chararray, state:chararray, pop:int);
-
- with_schema = FOREACH data {
- GENERATE city, state, pop, SchemaToString( (city, state, pop) );
- };
-
- STORE with_schema INTO 'output';
- */
- @Multiline private String onReadyTest;
-
- @Test
- public void onReadyTest() throws Exception
- {
- PigTest test = createPigTestFromString(onReadyTest);
-
- writeLinesToFile("input",
- "New York\tNY\t8244910",
- "Austin\tTX\t820611",
- "San Francisco\tCA\t812826");
-
- test.runScript();
-
- assertOutput(test, "with_schema",
- "(New York,NY,8244910,{(city: chararray,state: chararray,pop: int)})",
- "(Austin,TX,820611,{(city: chararray,state: chararray,pop: int)})",
- "(San Francisco,CA,812826,{(city: chararray,state: chararray,pop: int)})");
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/faab884c/datafu-pig/src/test/java/datafu/test/pig/util/SchemaToString.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/test/java/datafu/test/pig/util/SchemaToString.java b/datafu-pig/src/test/java/datafu/test/pig/util/SchemaToString.java
deleted file mode 100644
index e8714c4..0000000
--- a/datafu-pig/src/test/java/datafu/test/pig/util/SchemaToString.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.test.pig.util;
-
-import org.apache.pig.data.Tuple;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-
-import datafu.pig.util.SimpleEvalFunc;
-
-/**
- * Simple UDF for testing purposes.
- *
- * Captures a value at front-end (launch) time, namely a string for the input
- * schema, and returns its value (ignoring the actual input it characterizes)
- */
-public class SchemaToString extends SimpleEvalFunc<String> {
- private String schema_string = null;
-
- /**
- * Hook to capture the input schema string that we will spit out later
- */
- @Override
- public void onReady(Schema in_schema, Schema out_schema) {
- String sch_str = in_schema.toString();
- sch_str = sch_str.replaceFirst("[\\w\\.]+: ", "");
- getInstanceProperties().put("schema_string", sch_str);
- super.onReady(in_schema, out_schema);
- }
-
- /*
- * @param a tuple
- * @return the schema of the tuple
- */
- public String call(Tuple tup)
- {
- if (schema_string == null) { schema_string = (String)getInstanceProperties().get("schema_string"); }
- return schema_string;
- }
-}